From e1736ce9a49cbc5c0c28ed43b45f40befa47eb31 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 27 Oct 2026 23:48:59 -0500 Subject: [PATCH 01/50] Add experimental Vulkan GPU frame pipeline Work-in-progress Vulkan MVP, built only with -DENABLE_GPU_FRAME=ON. CPU builds are unchanged. - Frames can hold an immutable deferred GPU image and only read back CPU pixels when GetImage() is called. - FFmpegReader decodes through a shared Vulkan device and scales on the GPU. - Clip transforms and source-over compositing run in a Vulkan compute shader. - FFmpegWriter hands GPU frames to NVENC through Vulkan/CUDA interop. - VulkanOpenGLPresenter draws GPU frames in a QOpenGLWidget without readback. - Timeline and export benchmarks compare CPU and Vulkan builds. --- CMakeLists.txt | 1 + bindings/python/openshot.i | 6 + examples/CMakeLists.txt | 4 + examples/TimelineBenchmark.cpp | 59 +++++ examples/TimelineExportBenchmark.cpp | 66 +++++ examples/VulkanBenchmark.cpp | 11 +- examples/build-render-backend.sh | 30 +++ examples/run-timeline-benchmarks.py | 160 +++++++++++++ src/CMakeLists.txt | 39 +++ src/CacheMemory.cpp | 14 +- src/Clip.cpp | 71 +++++- src/FFmpegReader.cpp | 183 ++++++++++++++ src/FFmpegReader.h | 7 + src/FFmpegWriter.cpp | 56 ++++- src/Frame.cpp | 81 +++++++ src/Frame.h | 19 +- src/FrameImage.h | 25 ++ src/FrameMapper.cpp | 19 +- src/OpenShotVersion.h.in | 4 + src/Qt/VideoRenderer.cpp | 35 +++ src/Qt/VideoRenderer.h | 21 ++ src/Qt/VulkanOpenGLPresenter.cpp | 345 +++++++++++++++++++++++++++ src/Qt/VulkanOpenGLPresenter.h | 26 ++ src/RendererBase.cpp | 11 + src/RendererBase.h | 4 + src/Timeline.cpp | 6 + src/VulkanCompositor.cpp | 337 ++++++++++++++++++++++++++ src/VulkanCompositor.h | 18 ++ src/VulkanEncodeConverter.h | 11 + src/VulkanFrameImage.cpp | 118 +++++++++ src/VulkanFrameImage.h | 41 ++++ src/vulkan/affine.comp | 72 ++++++ src/vulkan/encode_nv12.comp | 28 +++ tests/CMakeLists.txt | 5 + tests/CacheMemory.cpp | 35 +++ tests/Clip.cpp | 52 ++++ tests/Frame.cpp | 76 ++++++ tests/FrameMapper.cpp | 41 ++++ tests/RendererBase.cpp | 98 ++++++++ tests/VulkanCompositor.cpp | 238 ++++++++++++++++++ tests/VulkanPresentation.cpp | 102 ++++++++ 41 files changed, 2562 insertions(+), 13 deletions(-) create mode 100644 examples/TimelineBenchmark.cpp create mode 100644 examples/TimelineExportBenchmark.cpp create mode 100644 examples/build-render-backend.sh create mode 100644 examples/run-timeline-benchmarks.py create mode 100644 src/FrameImage.h create mode 100644 src/Qt/VulkanOpenGLPresenter.cpp create mode 100644 src/Qt/VulkanOpenGLPresenter.h create mode 100644 src/VulkanCompositor.cpp create mode 100644 src/VulkanCompositor.h create mode 100644 src/VulkanEncodeConverter.h create mode 100644 src/VulkanFrameImage.cpp create mode 100644 src/VulkanFrameImage.h create mode 100644 src/vulkan/affine.comp create mode 100644 src/vulkan/encode_nv12.comp create mode 100644 tests/RendererBase.cpp create mode 100644 tests/VulkanCompositor.cpp create mode 100644 tests/VulkanPresentation.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2396801b6..745c685ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,7 @@ option(ENABLE_OPENCV "Build with OpenCV algorithms (requires Boost, Protobuf 3)" option(ENABLE_WAYLAND_CAPTURE "Build Wayland screen capture support with xdg-desktop-portal and PipeWire (Linux only)" ON) option(USE_HW_ACCEL "Enable hardware-accelerated encoding-decoding with FFmpeg 3.4+" ON) option(ENABLE_VULKAN_BENCHMARK "Build the experimental Vulkan benchmark example" OFF) +option(ENABLE_GPU_FRAME "Build experimental deferred GPU frame storage" OFF) # Legacy commandline override if (DISABLE_TESTS) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index 6602b465c..e6af31f13 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -110,6 +110,9 @@ class QWidget; #include "QtTextReader.h" #include "KeyFrame.h" #include "RendererBase.h" +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) +#include "Qt/VulkanOpenGLPresenter.h" +#endif #include "Settings.h" #include "TimelineBase.h" #include "Timeline.h" @@ -520,6 +523,9 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "KeyFrame.h" %include "AnimatedCurve.h" %include "RendererBase.h" +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) +%include "Qt/VulkanOpenGLPresenter.h" +#endif %include "Settings.h" %include "TimelineBase.h" %include "Qt/VideoCacheThread.h" diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 12e05cb9f..6a92d7dc6 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -64,6 +64,10 @@ target_compile_definitions(openshot-example PRIVATE target_link_libraries(openshot-example openshot) add_executable(openshot-html-example ExampleHtml.cpp) +add_executable(openshot-timeline-benchmark TimelineBenchmark.cpp) +target_link_libraries(openshot-timeline-benchmark openshot) +add_executable(openshot-export-benchmark TimelineExportBenchmark.cpp) +target_link_libraries(openshot-export-benchmark openshot) target_link_libraries(openshot-html-example openshot Qt${QT_VERSION_MAJOR}::Gui) if(ENABLE_VULKAN_BENCHMARK) diff --git a/examples/TimelineBenchmark.cpp b/examples/TimelineBenchmark.cpp new file mode 100644 index 000000000..b53324b37 --- /dev/null +++ b/examples/TimelineBenchmark.cpp @@ -0,0 +1,59 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "Clip.h" +#include "Timeline.h" +#include +#include +#include +#include +#include + +// Same workload for CPU and experimental GPU builds. Readback is optional so +// processing and the existing QImage presentation boundary can be distinguished. +int main(int argc, char** argv) { + if (argc < 2 || argc > 6) { + std::cerr << "Usage: openshot-timeline-benchmark VIDEO [FRAMES=120] [LAYERS=1] [READBACK=0] [DUMP.png]\n"; + return 2; + } + try { + const int count = argc > 2 ? std::stoi(argv[2]) : 120; + const int layers = argc > 3 ? std::stoi(argv[3]) : 1; + const bool readback = argc > 4 && std::stoi(argv[4]) != 0; + if (count < 1 || layers < 1 || layers > 16) throw std::invalid_argument("Invalid frame/layer count"); + std::vector> clips; + openshot::Timeline timeline(3840, 2160, openshot::Fraction(24, 1), 48000, 2, openshot::LAYOUT_STEREO); + timeline.SetMaxSize(854, 480); + for (int i = 0; i < layers; ++i) { + auto clip = std::make_unique(argv[1]); + clip->Layer(i); + if (i) { + clip->scale_x = openshot::Keyframe(0.7); + clip->scale_y = openshot::Keyframe(0.7); + clip->rotation = openshot::Keyframe(12.0 * i); + clip->alpha = openshot::Keyframe(0.6); + } + timeline.AddClip(clip.get()); + clips.push_back(std::move(clip)); + } + timeline.Open(); + int deferred = 0; + std::shared_ptr last; + const auto start = std::chrono::steady_clock::now(); + for (int i = 1; i <= count; ++i) { + last = timeline.GetFrame(i); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred += bool(last->GetDeferredImage()); +#endif + if (readback) last->GetImage(); + } + const double seconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + std::cout << "frames=" << count << " layers=" << layers << " readback=" << readback + << " deferred_frames=" << deferred << " seconds=" << seconds + << " fps=" << count / seconds << std::endl; + if (argc > 5 && !last->GetImage()->save(argv[5])) throw std::runtime_error("Dump failed"); + timeline.Close(); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/examples/TimelineExportBenchmark.cpp b/examples/TimelineExportBenchmark.cpp new file mode 100644 index 000000000..c61e80b2d --- /dev/null +++ b/examples/TimelineExportBenchmark.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "Clip.h" +#include "Timeline.h" +#include "FFmpegWriter.h" +#include +#include +#include +#include + +// Identical full-resolution rendering + NVENC workload for CPU/GPU builds. +// Optional audio input exercises timeline mixing and muxing. Output is replaced. +int main(int argc,char** argv) { + if(argc<3 || argc>6){ + std::cerr<<"Usage: openshot-export-benchmark VIDEO OUTPUT.mp4 [FRAMES=120] [LAYERS=2] [AUDIO]\n"; + return 2; + } + try { + const int count=argc>3?std::stoi(argv[3]):120; + const int layers=argc>4?std::stoi(argv[4]):2; + if(count<1 || layers<1 || layers>16) throw std::invalid_argument("Invalid frame/layer count"); + std::vector> clips; + openshot::Timeline timeline(3840,2160,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.SetMaxSize(3840,2160); + for(int index=0;index(argv[1]); + clip->Layer(index); + if(index){ + clip->scale_x=openshot::Keyframe(0.7); + clip->scale_y=openshot::Keyframe(0.65); + clip->shear_x=openshot::Keyframe(0.1); + clip->rotation=openshot::Keyframe(12.0*index); + clip->alpha=openshot::Keyframe(0.6); + } + timeline.AddClip(clip.get());clips.push_back(std::move(clip)); + } + if(argc>5){ + auto audio=std::make_unique(argv[5]);audio->Layer(-1); + // Do not render cover art embedded in an audio file. + audio->has_video=openshot::Keyframe(0.0); + timeline.AddClip(audio.get());clips.push_back(std::move(audio)); + } + timeline.Open(); + openshot::FFmpegWriter writer(argv[2]); + writer.SetVideoOptions(true,"h264_nvenc",openshot::Fraction(24,1),3840,2160,openshot::Fraction(1,1),false,false,40000000); + writer.SetAudioOptions(true,"aac",48000,2,openshot::LAYOUT_STEREO,192000); + writer.Open(); + int deferred=0; + std::shared_ptr last; + const auto start=std::chrono::steady_clock::now(); + for(int n=1;n<=count;++n){ + last=timeline.GetFrame(n); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred+=bool(last->GetDeferredImage()); +#endif + writer.WriteFrame(last); + } + writer.Close(); + const double seconds=std::chrono::duration(std::chrono::steady_clock::now()-start).count(); + std::cout<<"frames="<] [--overlay ] [--frames ]\n" << " [--mode ] [--vaapi-device ] [--row ]\n" << " [--dump-dir ] [--dump-every ]\n" + << " [--cpu-only | --vulkan-only] (select compositor)\n" << "Rows: CPU->CPU, CPU->Vk, VAAPI->CPU, VAAPI->Vk, CUDA->CPU, CUDA->Vk, Vulkan->CPU, VulkanDirect->Vk\n"; std::exit(0); } else { @@ -2312,7 +2313,9 @@ int main(int argc, char* argv[]) { const QImage overlay_image = LoadOverlayImage(options); std::vector results; auto row_enabled = [&](const std::string& name) { - return options.row_name.empty() || options.row_name == name; + const bool vulkan_composite = name.size() >= 4 && name.compare(name.size() - 4, 4, "->Vk") == 0; + return (vulkan_composite ? options.run_vulkan : options.run_cpu) && + (options.row_name.empty() || options.row_name == name); }; auto run_row = [&](BenchmarkResult (*fn)(const BenchmarkOptions&, const QImage&), const std::string& fallback_name) { @@ -2393,7 +2396,7 @@ int main(int argc, char* argv[]) { add_unavailable_row("VulkanDirect->Vk", "Vulkan runtime unavailable"); } if (results.empty()) - throw std::runtime_error("Unknown row name: " + options.row_name); + throw std::runtime_error("No rows selected: check --row and compositor filters"); for (const auto& result : results) { result.stats.Print(result.name + " / " + result.decode_backend + " / " + result.composite_backend); @@ -2406,6 +2409,10 @@ int main(int argc, char* argv[]) { std::cout << "\nMatrix Summary\n"; PrintResultTable(results); + const bool processed_frames = std::any_of(results.begin(), results.end(), + [](const BenchmarkResult& result) { return !result.skipped && result.stats.frames > 0; }); + if (!processed_frames) + throw std::runtime_error("No benchmark row successfully processed frames"); return 0; } catch (const std::exception& exc) { std::cerr << "Benchmark failed: " << exc.what() << "\n"; diff --git a/examples/build-render-backend.sh b/examples/build-render-backend.sh new file mode 100644 index 000000000..4e58dec3a --- /dev/null +++ b/examples/build-render-backend.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Build isolated CPU/Vulkan experiments without replacing installed libraries. +set -euo pipefail + +backend=${OPENSHOT_RENDER_BACKEND:-cpu} +case "$backend" in + cpu) vulkan=OFF; target=openshot-benchmark ;; + vulkan) vulkan=ON; target=openshot-vulkan-benchmark ;; + *) echo 'OPENSHOT_RENDER_BACKEND must be cpu or vulkan' >&2; exit 2 ;; +esac + +source_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +build_dir=${OPENSHOT_BUILD_DIR:-"$source_dir/build-mvp-$backend"} + +# Additional arguments are CMake dependency/toolchain settings. Backend and +# Release settings below take precedence so the two experiments stay comparable. +cmake -S "$source_dir" -B "$build_dir" "$@" \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_VULKAN_BENCHMARK="$vulkan" \ + -DENABLE_GPU_FRAME="$vulkan" \ + -DENABLE_PYTHON=ON \ + -DBUILD_TESTING=ON +cmake --build "$build_dir" --target "$target" openshot-timeline-benchmark pyopenshot --parallel "${OPENSHOT_BUILD_JOBS:-4}" + +if [[ "$backend" == vulkan ]]; then + echo "Built experimental Vulkan benchmark: $build_dir/examples/$target" + echo 'Experimental GPU library enabled; verify GPU residency and fallbacks with the timeline benchmark.' +else + echo "Built CPU library and benchmark: $build_dir/tests/$target" +fi diff --git a/examples/run-timeline-benchmarks.py b/examples/run-timeline-benchmarks.py new file mode 100644 index 000000000..b3bd6018d --- /dev/null +++ b/examples/run-timeline-benchmarks.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Run comparable CPU/GPU timeline workloads sequentially and retain raw evidence. + +Example (inherit the Vulkan ICD and FFmpeg library environment): + python3 examples/run-timeline-benchmarks.py --cpu-build build --vulkan-build build-vk \ + --video /path/to/4k.mp4 --output /tmp/timeline-results.json +""" +import argparse +import datetime +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import statistics +import subprocess +import sys + +RESULT = re.compile( + r"frames=(\d+) layers=(\d+) readback=(\d+) deferred_frames=(\d+) " + r"seconds=([\d.eE+-]+) fps=([\d.eE+-]+)" +) + + +def capture(command, timeout=20): + try: + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout) + return {"command": command, "returncode": result.returncode, "output": result.stdout} + except (OSError, subprocess.TimeoutExpired) as error: + return {"command": command, "error": str(error)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cpu-build", required=True, type=Path) + parser.add_argument("--vulkan-build", required=True, type=Path) + parser.add_argument("--video", required=True, type=Path) + parser.add_argument("--frames", default=120, type=int) + parser.add_argument("--runs", default=3, type=int) + parser.add_argument("--timeout", default=180, type=int) + parser.add_argument("--output", default=Path("timeline-results.json"), type=Path) + parser.add_argument("--note", default="", help="Record background services or other measurement conditions") + parser.add_argument("--verify-images", action="store_true", help="Save final-frame PNGs and compare CPU/GPU pixels (requires Pillow)") + args = parser.parse_args() + if min(args.frames, args.runs, args.timeout) < 1: + parser.error("frames, runs and timeout must be positive") + video = args.video.resolve(strict=True) + binaries = {name: (directory / "examples/openshot-timeline-benchmark").resolve(strict=True) + for name, directory in (("cpu", args.cpu_build), ("vulkan", args.vulkan_build))} + output = args.output.resolve() + if output == video or output in binaries.values(): + parser.error("output must not overwrite an input file") + repo = Path(__file__).resolve().parent.parent + report = { + "started_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "machine": {"platform": platform.platform(), "processor": platform.processor(), + "logical_cpus": os.cpu_count()}, + "video": {"path": str(video), "bytes": video.stat().st_size, + "mtime_ns": video.stat().st_mtime_ns}, + "frames": args.frames, "runs": args.runs, "measurement_note": args.note, + "environment": {name: os.environ[name] for name in ( + "VK_ICD_FILENAMES", "VK_DRIVER_FILES", "LD_LIBRARY_PATH", "QT_QPA_PLATFORM", + "OMP_NUM_THREADS") if name in os.environ}, + "revision": capture(["git", "-C", str(repo), "rev-parse", "HEAD"]), + "working_tree": capture(["git", "-C", str(repo), "status", "--short"]), + "binaries": {}, "measurements": [], "summary": [], + "notes": "Each trial is a fresh process; includes initialization. Run without other GPU/CPU workloads. " + "Readback=0 excludes presentation; readback=1 includes QImage materialization, not Qt display. " + "Compare library versions below before attributing differences to the backend." + } + for name, binary in binaries.items(): + report["binaries"][name] = {"path": str(binary), + "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "libraries": capture(["ldd", str(binary)])} + for name, command in (("gpu", ["nvidia-smi", "--query-gpu=name,driver_version,memory.total,memory.used,utilization.gpu", "--format=csv,noheader"]), + ("source", ["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(video)])): + if shutil.which(command[0]): + report[name] = capture(command) + + if shutil.which("nvidia-smi"): + report["gpu_processes"] = capture(["nvidia-smi", "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader"]) + + def save(): + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2) + "\n") + + if args.verify_images: + from PIL import Image, ImageChops, ImageStat + image_dir = output.parent / (output.stem + "-frames") + image_dir.mkdir(parents=True, exist_ok=True) + cpu_images = {} + failed = False + for layers in (1, 2): + for readback in (0, 1): + for backend, binary in binaries.items(): + successes = [] + for trial in range(1, args.runs + 1): + command = [str(binary), str(video), str(args.frames), str(layers), str(readback)] + image_path = None + if args.verify_images: + image_path = image_dir / f"{backend}-layers{layers}-readback{readback}-run{trial}.png" + command.append(str(image_path)) + evidence = capture(command, args.timeout) + row = {"backend": backend, "layers": layers, "readback": readback, + "trial": trial, **evidence} + match = RESULT.search(evidence.get("output", "")) + if evidence.get("returncode") == 0 and match: + frames, got_layers, got_readback, deferred, seconds, fps = match.groups() + row.update(frames=int(frames), deferred_frames=int(deferred), + seconds=float(seconds), fps=float(fps)) + row["valid"] = (int(frames) == args.frames and int(got_layers) == layers + and int(got_readback) == readback and float(seconds) > 0 + and (backend != "vulkan" or int(deferred) == args.frames)) + if not row["valid"]: + row["error"] = "Incomplete workload or Vulkan CPU fallback detected" + else: + row["valid"] = False + if row["valid"] and image_path: + try: + with Image.open(image_path) as loaded: + pixels = loaded.convert("RGB") + means = ImageStat.Stat(pixels).mean + row["image"] = {"path": str(image_path), "size": pixels.size, "mean_rgb": means} + if backend == "cpu": + cpu_images[(layers, readback)] = pixels + else: + reference = cpu_images[(layers, readback)] + if pixels.size != reference.size: + raise ValueError("GPU image dimensions differ from CPU") + row["image"]["cpu_mae_rgb"] = ImageStat.Stat(ImageChops.difference(pixels, reference)).mean + reference_means = ImageStat.Stat(reference).mean + if sum(reference_means) > 3 and sum(means) < 0.1 * sum(reference_means): + raise ValueError("GPU image is black/near-black while matching CPU image is not") + except (OSError, ValueError) as error: + row["valid"] = False + row["error"] = str(error) + failed |= not row["valid"] + if row["valid"]: + successes.append(row["fps"]) + report["measurements"].append(row) + print(f"{backend} layers={layers} readback={readback} trial={trial}: " + f"{row.get('fps', 'no result')} FPS valid={row['valid']}", flush=True) + save() + report["summary"].append({"backend": backend, "layers": layers, "readback": readback, + "valid_runs": len(successes), + "median_fps": statistics.median(successes) if successes else None, + "min_fps": min(successes) if successes else None, + "max_fps": max(successes) if successes else None}) + report["finished_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + report["valid"] = not failed + save() + print(f"Results: {output}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87867c6aa..258465259 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -185,6 +185,9 @@ set(CMAKE_MACOSX_RPATH 0) ############### CREATE LIBRARY ################# # Create shared openshot library add_library(openshot SHARED) +if(ENABLE_GPU_FRAME) + target_compile_definitions(openshot PUBLIC OPENSHOT_ENABLE_GPU_FRAME=1) +endif() target_sources(openshot PRIVATE ${OPENSHOT_SOURCES} @@ -441,6 +444,42 @@ find_package(FFmpeg REQUIRED ) set(all_comps avcodec avdevice avformat avutil swscale) +if(ENABLE_GPU_FRAME) + find_package(FFmpeg REQUIRED COMPONENTS avfilter) + list(APPEND all_comps avfilter) + target_sources(openshot PRIVATE VulkanFrameImage.cpp) + target_sources(openshot PRIVATE VulkanCompositor.cpp) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_sources(openshot PRIVATE Qt/VulkanOpenGLPresenter.cpp Qt/VulkanOpenGLPresenter.h) + endif() + find_package(Vulkan REQUIRED) + find_program(GLSLANG_VALIDATOR glslangValidator) + if(NOT GLSLANG_VALIDATOR) + message(FATAL_ERROR "ENABLE_GPU_FRAME requires glslangValidator") + endif() + set(VULKAN_AFFINE_SHADER "${CMAKE_CURRENT_BINARY_DIR}/affine.comp.spv") + add_custom_command(OUTPUT "${VULKAN_AFFINE_SHADER}" + COMMAND "${GLSLANG_VALIDATOR}" -V -S comp -o "${VULKAN_AFFINE_SHADER}" + "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/affine.comp" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/affine.comp" VERBATIM) + add_custom_target(openshot-vulkan-shaders DEPENDS "${VULKAN_AFFINE_SHADER}") + set(VULKAN_ENCODE_SHADER "${CMAKE_CURRENT_BINARY_DIR}/encode_nv12.comp.spv") + add_custom_command(OUTPUT "${VULKAN_ENCODE_SHADER}" + COMMAND "${GLSLANG_VALIDATOR}" -V -S comp -o "${VULKAN_ENCODE_SHADER}" + "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/encode_nv12.comp" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/encode_nv12.comp" VERBATIM) + add_custom_target(openshot-vulkan-encode-shaders DEPENDS "${VULKAN_ENCODE_SHADER}") + add_dependencies(openshot openshot-vulkan-encode-shaders) + target_compile_definitions(openshot PRIVATE OPENSHOT_VULKAN_ENCODE_SHADER_PATH="${VULKAN_ENCODE_SHADER}") + add_dependencies(openshot openshot-vulkan-shaders) + target_compile_definitions(openshot PRIVATE OPENSHOT_VULKAN_AFFINE_SHADER_PATH="${VULKAN_AFFINE_SHADER}") + target_link_libraries(openshot PRIVATE Vulkan::Vulkan) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + pkg_check_modules(VULKAN_FRAME_PC QUIET vulkan) + target_include_directories(openshot PRIVATE ${VULKAN_FRAME_PC_INCLUDE_DIRS}) + endif() +endif() set(version_comps avcodec avformat avutil) # Pick a resampler. Prefer swresample if possible diff --git a/src/CacheMemory.cpp b/src/CacheMemory.cpp index a89c9f7ee..1b7758e1f 100644 --- a/src/CacheMemory.cpp +++ b/src/CacheMemory.cpp @@ -255,7 +255,19 @@ void CacheMemory::CleanUp() // Create a scoped lock, to protect the cache from multiple threads const std::lock_guard lock(*cacheMutex); - while (GetBytes() > max_bytes && frame_numbers.size() > 20) + size_t retained_frames = 20; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // A CPU-oriented twenty-frame floor can pin gigabytes of GPU memory + // across reader, mapper and timeline caches. Honor byte budgets for + // caches holding hardware surfaces while retaining the newest frame. + for (const auto& entry : frames) { + if (entry.second->GetDeferredImage()) { + retained_frames = 1; + break; + } + } +#endif + while (GetBytes() > max_bytes && frame_numbers.size() > retained_frames) { // Get the oldest frame number. int64_t frame_to_remove = frame_numbers.back(); diff --git a/src/Clip.cpp b/src/Clip.cpp index 0fc043d87..7a06f32ab 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -17,6 +17,10 @@ #include "Exceptions.h" #include "FFmpegReader.h" #include "FrameMapper.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#endif #include "QtImageReader.h" #include "QtTextReader.h" #include "ChunkReader.h" @@ -503,6 +507,15 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun // Get time mapped frame object (used to increase speed, change direction, etc...) apply_timemapping(frame); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Audio-only/disabled-video layers have nothing to paint in the basic + // pipeline. Keep their audio, without manufacturing a CPU canvas. + if (options && (!reader->info.has_video || has_video.GetInt(frame->number) == 0) && + !Waveform() && effects.empty() && + (!timeline || static_cast(timeline)->Effects().empty())) + return frame; +#endif + // Apply waveform image (if any) apply_waveform(frame, timeline_size); @@ -1352,6 +1365,34 @@ void Clip::RemoveEffect(EffectBase* effect) void Clip::apply_background(std::shared_ptr frame, std::shared_ptr background_frame, bool update_frame_image) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Audio-only frames contribute audio in Timeline::add_layer, but have no + // image to composite. Do not materialize the GPU canvas for an empty layer. + if (!frame->has_image_data) { + if (update_frame_image) { + if (auto background = background_frame->GetDeferredImage()) frame->AddDeferredImage(background); + else frame->AddImage(background_frame->GetImage()); + } + return; + } + if (composite == QPainter::CompositionMode_SourceOver) { + auto source = frame->GetDeferredImage(); + auto background = background_frame->GetDeferredImage(); + QColor clear; + if (std::dynamic_pointer_cast(source) && + (std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear))) { + try { + auto result = RenderVulkanImage(source, background, background_frame->GetWidth(), + background_frame->GetHeight(), QTransform(), 1.0, clear); + background_frame->AddDeferredImage(result); + if (update_frame_image) frame->AddDeferredImage(result); + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan composite CPU fallback: ") + error.what()); + } + } + } +#endif // Add background canvas std::shared_ptr background_canvas = background_frame->GetImage(); QPainter painter(background_canvas.get()); @@ -1402,6 +1443,22 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { return; } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (composite == QPainter::CompositionMode_SourceOver && + corner_radius.GetValue(frame->number) == 0.0 && display == FRAME_DISPLAY_NONE) { + if (auto source = std::dynamic_pointer_cast(frame->GetDeferredImage())) { + try { + frame->AddDeferredImage(RenderVulkanImage(source, nullptr, timeline_size.width(), + timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()), + alpha.GetValue(frame->number))); + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan transform CPU fallback: ") + error.what()); + } + } + } +#endif + // Get image from clip, and create transparent background image std::shared_ptr source_image = frame->GetImage(); std::shared_ptr background_canvas = std::make_shared(timeline_size.width(), @@ -1557,8 +1614,12 @@ QSize Clip::scale_size(QSize source_size, ScaleType source_scale, int target_wid // Get QTransform from keyframes QTransform Clip::get_transform(std::shared_ptr frame, int width, int height) { - // Get image from clip - std::shared_ptr source_image = frame->GetImage(); + // Transform geometry does not require CPU pixels for a deferred surface. +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const QSize source_dimensions(frame->GetWidth(), frame->GetHeight()); +#else + const QSize source_dimensions = frame->GetImage()->size(); +#endif const double margin_value = std::max(0.0, std::min(0.5, margin.GetValue(frame->number))); const double margin_pixels = margin_value * std::min(width, height); @@ -1568,7 +1629,7 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig const double layout_height = std::max(1.0, height - (margin_pixels * 2.0)); /* RESIZE SOURCE IMAGE - based on scale type */ - QSize source_size = scale_size(source_image->size(), scale, layout_width, layout_height); + QSize source_size = scale_size(source_dimensions, scale, layout_width, layout_height); // Text rasterizes extra detail for a scaled-up SCALE_NONE clip; draw it at its layout size if (scale == SCALE_NONE) { @@ -1758,8 +1819,8 @@ QTransform Clip::get_transform(std::shared_ptr frame, int width, int heig transform.translate(-origin_x_offset,-origin_y_offset); } // SCALE CLIP (if needed) - float source_width_scale = (float(source_size.width()) / float(source_image->width())) * sx; - float source_height_scale = (float(source_size.height()) / float(source_image->height())) * sy; + float source_width_scale = (float(source_size.width()) / float(source_dimensions.width())) * sx; + float source_height_scale = (float(source_size.height()) / float(source_dimensions.height())) * sy; if (!isNear(source_width_scale, 1.0) || !isNear(source_height_scale, 1.0)) { transform.scale(source_width_scale, source_height_scale); } diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 842bf69fb..5ab2c9bb8 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -28,6 +28,9 @@ #include "effects/CropHelpers.h" #include "FFmpegReader.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanFrameImage.h" +#endif #include "Exceptions.h" #include "MemoryTrim.h" #include "Timeline.h" @@ -139,9 +142,37 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 #if USE_HW_ACCEL +#ifdef OPENSHOT_ENABLE_GPU_FRAME +// All readers share one device so their surfaces can be composed without interop. +// Keep an owning reference for the process lifetime; each reader receives its own ref. +static int CreateVulkanDecodeDevice(AVBufferRef** output) +{ + struct SharedDevice { + std::mutex mutex; + AVBufferRef* device = nullptr; + ~SharedDevice() { av_buffer_unref(&device); } + }; + static SharedDevice shared; + const std::lock_guard lock(shared.mutex); + if (!shared.device) { + int result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0); + if (result < 0) return result; + } + *output = av_buffer_ref(shared.device); + return *output ? 0 : AVERROR(ENOMEM); +} +#endif + // Get hardware pix format static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VULKAN) { + for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format) + if (*format == AV_PIX_FMT_VULKAN) return *format; + return avcodec_default_get_format(ctx, pix_fmts); + } +#endif const enum AVPixelFormat *p; // Prefer only the format matching the selected hardware decoder @@ -247,6 +278,10 @@ void FFmpegReader::Open() { pFormatCtx = NULL; { hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_on = !force_sw_decode; + vulkan_processor.reset(); +#endif hw_decode_failed = false; hw_decode_error_count = 0; hw_decode_succeeded = false; @@ -309,6 +344,16 @@ void FFmpegReader::Open() { if (hw_de_on && (retry_decode_open==2)) { // Up to here no decision is made if hardware or software decode hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_supported = 0; + for (int index = 0; const AVCodecHWConfig* config = avcodec_get_hw_config(pCodec, index); ++index) { + if (config->device_type == AV_HWDEVICE_TYPE_VULKAN && + (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) { + hw_de_supported = 1; + break; + } + } +#endif } #endif retry_decode_open = 0; @@ -324,6 +369,11 @@ void FFmpegReader::Open() { av_dict_set(&opts, "strict", "experimental", 0); #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Vulkan decode surfaces must not be held by frame-thread workers + // while the consumer waits on their timeline semaphores. + pCodecCtx->thread_type = FF_THREAD_SLICE; +#endif // Open Hardware Acceleration int i_decoder_hw = 0; char adapter[256]; @@ -413,9 +463,17 @@ void FFmpegReader::Open() { Logger::Instance()->AppendDebugMethod("Decode Device not present using default"); } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN; + adapter_ptr = nullptr; +#endif hw_device_ctx = NULL; // Here the first hardware initialisations are made + #ifdef OPENSHOT_ENABLE_GPU_FRAME + if (CreateVulkanDecodeDevice(&hw_device_ctx) >= 0) { +#else if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { +#endif const char* hw_name = av_hwdevice_get_type_name(hw_de_av_device_type); std::string hw_msg = "HW decode active: "; hw_msg += (hw_name ? hw_name : "unknown"); @@ -453,7 +511,12 @@ void FFmpegReader::Open() { } else { Logger::Instance()->Log("HW decode active: no (falling back to software)"); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_supported = 0; + pCodecCtx->get_format = avcodec_default_get_format; +#else throw InvalidCodec("Hardware device create failed.", path); +#endif } } #endif // USE_HW_ACCEL @@ -485,6 +548,9 @@ void FFmpegReader::Open() { ((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config; constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig); #endif // ENABLE_VAAPI +#ifdef OPENSHOT_ENABLE_GPU_FRAME + constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx, nullptr); +#endif if (constraints) { if (pCodecCtx->coded_width < constraints->min_width || pCodecCtx->coded_height < constraints->min_height || @@ -703,6 +769,16 @@ void FFmpegReader::Open() { const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2); working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels); final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (hw_de_on && hw_de_supported) { + // Full-resolution hardware images consume device memory, not RAM. + // Bound each reader cache independently of FPS/thread-count heuristics. + constexpr int64_t gpu_cache_bytes = 256LL * 1024 * 1024; + // Pending video/audio frames cannot be evicted before they are paired. + // Keep the existing working-cache budget; bound finalized images below. + final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), gpu_cache_bytes)); + } +#endif // Scan PTS for any offsets (i.e. non-zero starting streams). At least 1 stream must start at zero timestamp. // This method allows us to shift timestamps to ensure at least 1 stream is starting at zero. @@ -733,6 +809,9 @@ void FFmpegReader::Close() { // Mark as "closed" is_open = false; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + vulkan_processor.reset(); +#endif // Keep track of most recent packet AVPacket *recent_packet = packet; @@ -1460,6 +1539,10 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { // Copy and return the largest processed frame (assuming it was the last in the video file) std::shared_ptr f = CreateFrame(largest_frame_processed); if (frame->has_image_data) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(frame->GetImage()->copy())); } @@ -1478,9 +1561,17 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { std::shared_ptr f = CreateFrame(largest_frame_processed); if (last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= requested_frame) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); } else if (last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= requested_frame) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); } else { f->AddColor(info.width, info.height, "#000"); @@ -1565,8 +1656,13 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL // Get the format from the variables set in get_hw_dec_format +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_de_av_pix_fmt = AV_PIX_FMT_VULKAN; + hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN; +#else hw_de_av_pix_fmt = hw_de_av_pix_fmt_global; hw_de_av_device_type = hw_de_av_device_type_global; +#endif #endif // USE_HW_ACCEL if (send_packet_err < 0 && send_packet_err != AVERROR_EOF) { Logger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (send packet: Not sent [" + av_err2string(send_packet_err) + "])", "send_packet_err", send_packet_err, "send_packet_pts", send_packet_pts); @@ -1634,6 +1730,11 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { int err; + #ifdef OPENSHOT_ENABLE_GPU_FRAME + if (next_frame2->format == AV_PIX_FMT_VULKAN) { + decoded_frame = next_frame2; + } else +#endif if (next_frame2->format == hw_de_av_pix_fmt) { if ((err = av_hwframe_transfer_data(next_frame, next_frame2, 0)) < 0) { Logger::Instance()->AppendDebugMethod( @@ -1688,7 +1789,11 @@ bool FFmpegReader::GetAVFrame() { hw_decode_error_count = 0; #if USE_HW_ACCEL if (hw_de_on && hw_de_supported && !force_sw_decode) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + hw_decode_succeeded = hw_decode_succeeded || next_frame2->format == AV_PIX_FMT_VULKAN; +#else hw_decode_succeeded = true; +#endif } #endif packet_status.video_decoded++; @@ -1699,6 +1804,12 @@ bool FFmpegReader::GetAVFrame() { decoded_pix_fmt = (AVPixelFormat)(pStream->codecpar->format); const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width; const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (decoded_pix_fmt == AV_PIX_FMT_VULKAN) { + if (av_frame_ref(pFrame, decoded_frame) < 0) + throw OutOfMemory("Failed to retain Vulkan frame", path); + } else { +#endif if (AV_ALLOCATE_IMAGE(pFrame, decoded_pix_fmt, decoded_width, decoded_height) <= 0) { throw OutOfMemory("Failed to allocate image buffer", path); } @@ -1712,6 +1823,9 @@ bool FFmpegReader::GetAVFrame() { pFrame->color_primaries = decoded_frame->color_primaries; pFrame->color_trc = decoded_frame->color_trc; pFrame->chroma_location = decoded_frame->chroma_location; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + } +#endif // Get display PTS from video frame, often different than packet->pts. // Sending packets to the decoder (i.e. packet->pts) is async, @@ -2018,6 +2132,32 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (pFrame && pFrame->format == AV_PIX_FMT_VULKAN) { + try { + if (!vulkan_processor) vulkan_processor = std::make_shared(); + auto f = CreateFrame(current_frame); + f->AddDeferredImage(vulkan_processor->Process(pFrame, width, height, openshot::Settings::Instance()->HIGH_QUALITY_SCALING)); + if (ApplyOrientationMetadata() && std::fabs(source_rotation) >= 0.0001) + Logger::Instance()->AppendDebugMethod("Vulkan reader: source orientation requires scaled CPU readback"); + ApplyFrameOrientation(f); + if (first_video_frame == 0 && !is_seeking) first_video_frame = current_frame; + working_cache.Add(f); + last_video_frame = f; + RemoveAVFrame(pFrame); + pFrame = nullptr; + video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds; + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan processing failed; retrying software decode: ") + error.what()); + hw_decode_failed = true; + RemoveAVFrame(pFrame); + pFrame = nullptr; + return; + } + } +#endif + // Determine required buffer size and allocate buffer const int bytes_per_pixel = 4; int raw_buffer_size = (width * height * bytes_per_pixel) + 128; @@ -2821,6 +2961,10 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { // visual order, especially when decode/prefetch is out-of-order. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); } @@ -2829,6 +2973,10 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { && last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= f->number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); } @@ -2837,6 +2985,10 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { && last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= f->number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); } @@ -2888,18 +3040,30 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { // recent image so playback does not stall waiting for video EOF. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); } if (!f->has_image_data && last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= f->number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); } if (!f->has_image_data && last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= f->number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); } } @@ -2910,18 +3074,30 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (!f->has_image_data && is_video_ready && is_audio_ready) { std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); } if (!f->has_image_data && last_final_video_frame && last_final_video_frame->has_image_data && last_final_video_frame->number <= f->number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); } if (!f->has_image_data && last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= f->number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); + else +#endif f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); } } @@ -3049,6 +3225,13 @@ void FFmpegReader::CheckFPS() { void FFmpegReader::RemoveAVFrame(AVFrame *remove_frame) { // Remove pFrame (if exists) if (remove_frame) { + // Referenced hardware frames release their image through AVBufferRef. +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (remove_frame->hw_frames_ctx) { + av_frame_free(&remove_frame); + return; + } +#endif // Free memory av_freep(&remove_frame->data[0]); #ifndef WIN32 diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index c8f1abfce..5c5765697 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -36,6 +36,9 @@ namespace openshot { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + class VulkanFrameProcessor; +#endif /** * @brief This struct holds the packet counts and end-of-file detection for an openshot::FFmpegReader. * @@ -103,6 +106,10 @@ namespace openshot { class FFmpegReader : public ReaderBase { private: std::string path; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::shared_ptr vulkan_processor; + int hw_de_on = 0; // Per-reader state for concurrent Vulkan/CPU readers. +#endif AVFormatContext *pFormatCtx; int videoStream, audioStream; diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index fd66b0023..229143792 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -29,6 +29,11 @@ #include "FFmpegWriter.h" #include "Exceptions.h" #include "Frame.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanFrameImage.h" +#include "VulkanCompositor.h" +#include "VulkanEncodeConverter.h" +#endif #include "OpenMPUtilities.h" #include "Settings.h" #include "Logger.h" @@ -1145,6 +1150,9 @@ void FFmpegWriter::write_frame(std::shared_ptr frame) { } // Deallocate buffer and AVFrame +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (!frame_final->hw_frames_ctx) +#endif av_freep(&(frame_final->data[0])); AV_FREE_FRAME(&frame_final); av_frames.erase(frame); @@ -2610,6 +2618,42 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { if (src_w == 1 && src_h == 1) return; +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL + // NVENC accepts encoder-owned CUDA surfaces. Preserve GPU residency through + // RGB->NV12 conversion and the Vulkan/CUDA device-to-device transfer. + if (hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx && + video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA) { + auto surface = frame->GetDeferredImage(); + if (surface) { + AVFrame* encoded = nullptr; + try { + auto native = std::dynamic_pointer_cast(surface); + const auto* context = native ? reinterpret_cast(native->GetFrame()->hw_frames_ctx->data) : nullptr; + if (src_w != info.width || src_h != info.height || !context || context->sw_format != AV_PIX_FMT_RGBA) { + QTransform scale; + scale.scale(double(info.width) / src_w, double(info.height) / src_h); + surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black); + } + auto nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface)); + encoded = av_frame_alloc(); + if (!encoded) throw std::bad_alloc(); + int status = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); + if (status >= 0) status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0); + if (status < 0) throw std::runtime_error("Vulkan/CUDA export transfer failed: " + av_err2string(status)); + av_frame_copy_props(encoded, nv12->GetFrame()); + add_avframe(frame, encoded); + Logger::Instance()->AppendDebugMethod("Vulkan export GPU transfer to NVENC", "frame", frame->number); + return; + } catch (...) { + av_frame_free(&encoded); + // Failed interop may leave external semaphore ownership uncertain. + // Surface the error instead of attempting CPU readback on that surface. + throw; + } + } + } +#endif + // Point persistent_src_frame->data to RGBA pixels const uchar* pixels = frame->GetPixels(); if (!persistent_src_frame) { @@ -2798,7 +2842,11 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra // Assign the initial AVFrame PTS from the frame counter frame_final->pts = video_timestamp; #if USE_HW_ACCEL - if (hw_en_on && hw_en_supported) { + if (hw_en_on && hw_en_supported +#ifdef OPENSHOT_ENABLE_GPU_FRAME + && !frame_final->hw_frames_ctx +#endif + ) { if (!(hw_frame = av_frame_alloc())) { std::clog << "Error code: av_hwframe_alloc\n"; } @@ -2824,7 +2872,11 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra #if USE_HW_ACCEL if (hw_en_on && hw_en_supported) { - ret = avcodec_send_frame(video_codec_ctx, hw_frame); //hw_frame!!! + #ifdef OPENSHOT_ENABLE_GPU_FRAME + ret = avcodec_send_frame(video_codec_ctx, frame_final->hw_frames_ctx ? frame_final : hw_frame); +#else + ret = avcodec_send_frame(video_codec_ctx, hw_frame); +#endif } else #endif // USE_HW_ACCEL { diff --git a/src/Frame.cpp b/src/Frame.cpp index 1862a82d0..b47adcef6 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -14,6 +14,7 @@ #include // for std::chrono::milliseconds #include #include +#include #include "Frame.h" #include "AudioBufferSource.h" @@ -86,6 +87,13 @@ Frame& Frame::operator= (const Frame& other) // Copy data and pointers from another Frame instance void Frame::DeepCopy(const Frame& other) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (this == &other) + return; + std::scoped_lock lock(addingImageMutex, other.addingImageMutex); + deferred_image = other.deferred_image; + image.reset(); +#endif number = other.number; capture_timestamp = other.capture_timestamp; channels = other.channels; @@ -381,11 +389,18 @@ juce::AudioBuffer *Frame::GetAudioSampleBuffer() // Get the size in bytes of this frame (rough estimate) int64_t Frame::GetBytes() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const std::lock_guard lock(addingImageMutex); +#endif int64_t total_bytes = 0; if (image) { total_bytes += static_cast( width * height * sizeof(char) * 4); } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (deferred_image) + total_bytes += deferred_image->GetBytes(); +#endif if (audio) { // approximate audio size (sample rate / 24 fps) total_bytes += (sample_rate / 24.0) * sizeof(float); @@ -398,6 +413,9 @@ int64_t Frame::GetBytes() // Get pixel data (as packets) const unsigned char* Frame::GetPixels() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + GetImage(); +#endif // Check for blank image if (!image) // Fill with black @@ -410,6 +428,9 @@ const unsigned char* Frame::GetPixels() // Get pixel data (for only a single scan-line) const unsigned char* Frame::GetPixels(int row) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + GetImage(); +#endif // Check for blank image if (!image) // Fill with black @@ -421,6 +442,9 @@ const unsigned char* Frame::GetPixels(int row) // Check a specific pixel color value (returns True/False) bool Frame::CheckPixel(int row, int col, int red, int green, int blue, int alpha, int threshold) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (deferred_image) GetImage(); +#endif int col_pos = col * 4; // Find column array position if (!image || row < 0 || row >= (height - 1) || col_pos < 0 || col_pos >= (width - 1) ) { @@ -703,6 +727,9 @@ void Frame::AddColor(const QColor& new_color) { // Create new image object, and fill with pixel data const std::lock_guard lock(addingImageMutex); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred_image.reset(); +#endif image = std::make_shared(width, height, QImage::Format_RGBA8888_Premultiplied); // Fill with solid color @@ -741,6 +768,9 @@ void Frame::AddImage(std::shared_ptr new_image) // assign image data const std::lock_guard lock(addingImageMutex); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + deferred_image.reset(); +#endif image = new_image; // Always convert to Format_RGBA8888_Premultiplied (if different) @@ -760,6 +790,9 @@ void Frame::AddImage(std::shared_ptr new_image, bool only_odd_lines) if (!new_image) return; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (deferred_image) GetImage(); +#endif // Check for blank source image if (!image) { // Replace the blank source image @@ -871,6 +904,17 @@ void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, // Get pointer to Magick++ image object std::shared_ptr Frame::GetImage() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const std::lock_guard lock(addingImageMutex); + if (deferred_image) { + auto materialized = deferred_image->Materialize(); + if (materialized.isNull() || materialized.width() != width || materialized.height() != height) + throw std::runtime_error("Deferred frame image returned invalid dimensions"); + // GetImage exposes mutable pixels. Make CPU storage authoritative so a + // later GPU consumer cannot accidentally reuse stale surface contents. + AddImage(std::make_shared(std::move(materialized))); + } +#endif // Check for blank image if (!image) // Fill with black @@ -895,6 +939,9 @@ cv::Mat Frame::Qimage2mat( std::shared_ptr& qimage) { // Get pointer to OpenCV image object cv::Mat Frame::GetImageCV() { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + GetImage(); +#endif // Check for blank image if (!image) // Fill with black @@ -924,7 +971,11 @@ std::shared_ptr Frame::Mat2Qimage(cv::Mat img){ void Frame::SetImageCV(cv::Mat _image) { imagecv = _image; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + AddImage(Mat2Qimage(_image)); +#else image = Mat2Qimage(_image); +#endif } #endif @@ -1015,3 +1066,33 @@ void Frame::AddAudioSilence(int numSamples) // Reset audio direction audio_is_increasing = true; } + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +void Frame::AddDeferredImage(std::shared_ptr new_image) +{ + if (!new_image) + return; + if (new_image->GetWidth() <= 0 || new_image->GetHeight() <= 0 || new_image->GetBytes() < 0) + throw std::invalid_argument("Invalid deferred frame image"); + const std::lock_guard lock(addingImageMutex); + width = new_image->GetWidth(); + height = new_image->GetHeight(); + image.reset(); + deferred_image = std::move(new_image); + has_image_data = true; +} + +std::shared_ptr Frame::GetDeferredImage() const +{ + const std::lock_guard lock(addingImageMutex); + return deferred_image; +} + +bool Frame::GetBlankColor(QColor& result) const +{ + const std::lock_guard lock(addingImageMutex); + if (image || deferred_image) return false; + result = QColor(QString::fromStdString(color)); + return result.isValid(); +} +#endif diff --git a/src/Frame.h b/src/Frame.h index 82106f685..bc440a53e 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -13,6 +13,8 @@ #ifndef OPENSHOT_FRAME_H #define OPENSHOT_FRAME_H +#include "OpenShotVersion.h" + #ifdef USE_OPENCV #define int64 opencv_broken_int #define uint64 opencv_broken_uint @@ -27,6 +29,9 @@ #include "ChannelLayouts.h" #include "Enums.h" #include "Fraction.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "FrameImage.h" +#endif #include #include @@ -90,10 +95,13 @@ namespace openshot { private: std::shared_ptr image; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::shared_ptr deferred_image; +#endif std::shared_ptr wave_image; std::shared_ptr previewApp; - std::recursive_mutex addingImageMutex; + mutable std::recursive_mutex addingImageMutex; std::recursive_mutex addingAudioMutex; openshot::Fraction pixel_ratio; int channels; @@ -120,6 +128,15 @@ namespace openshot bool has_image_data; ///< This frame has been loaded with pixel data +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && !defined(SWIG) + /// Attach an immutable deferred image without materializing CPU pixels. + void AddDeferredImage(std::shared_ptr new_image); + /// Returns the surface only while it remains authoritative. + std::shared_ptr GetDeferredImage() const; + /// Report an unmaterialized solid canvas without allocating CPU pixels. + bool GetBlankColor(QColor& result) const; +#endif + /// Constructor - blank frame Frame(); diff --git a/src/FrameImage.h b/src/FrameImage.h new file mode 100644 index 000000000..199e89f02 --- /dev/null +++ b/src/FrameImage.h @@ -0,0 +1,25 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_FRAME_IMAGE_H +#define OPENSHOT_FRAME_IMAGE_H + +#include +#include + +namespace openshot { + +/// Immutable image storage, including a hardware surface with deferred readback. +/// Implementations own their resources and synchronization. Materialize must be +/// safe for concurrent callers and return independent/COW CPU image storage. +class FrameImage { +public: + virtual ~FrameImage() = default; + virtual int GetWidth() const = 0; + virtual int GetHeight() const = 0; + /// Conservative resident allocation size, for cache budgeting. + virtual int64_t GetBytes() const = 0; + virtual QImage Materialize() const = 0; +}; + +} // namespace openshot +#endif diff --git a/src/FrameMapper.cpp b/src/FrameMapper.cpp index eb5164163..c5031045b 100644 --- a/src/FrameMapper.cpp +++ b/src/FrameMapper.cpp @@ -52,6 +52,9 @@ FrameMapper::FrameMapper(ReaderBase *reader, Fraction target, PulldownType targe // Adjust cache size based on size of frame and audio const int initial_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS); final_cache.SetMaxBytesFromInfo(initial_cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), 128LL * 1024 * 1024)); +#endif } // Destructor @@ -548,8 +551,17 @@ std::shared_ptr FrameMapper::GetFrame(int64_t requested_frame) // Copy the image from the odd field std::shared_ptr odd_frame = mapped_frame; - if (odd_frame && odd_frame->has_image_data) - frame->AddImage(std::make_shared(*odd_frame->GetImage()), true); + if (odd_frame && odd_frame->has_image_data) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Progressive mapping needs no pixel access. Preserve immutable GPU + // storage while remapping the frame number and audio independently. + auto surface = odd_frame->GetDeferredImage(); + if (surface && mapped.Odd.Frame == mapped.Even.Frame) + frame->AddDeferredImage(std::move(surface)); + else +#endif + frame->AddImage(std::make_shared(*odd_frame->GetImage()), true); + } if (mapped.Odd.Frame != mapped.Even.Frame) { // Add even lines (if different than the previous image) std::shared_ptr even_frame; @@ -858,6 +870,9 @@ void FrameMapper::ChangeMapping(Fraction target_fps, PulldownType target_pulldow // Adjust cache size based on size of frame and audio const int reset_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4); final_cache.SetMaxBytesFromInfo(reset_cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache.SetMaxBytes(std::min(final_cache.GetMaxBytes(), 128LL * 1024 * 1024)); +#endif // Deallocate resample buffer if (avr) { diff --git a/src/OpenShotVersion.h.in b/src/OpenShotVersion.h.in index 7452cba01..3c03d9212 100644 --- a/src/OpenShotVersion.h.in +++ b/src/OpenShotVersion.h.in @@ -34,6 +34,10 @@ #cmakedefine01 HAVE_OPENCV #cmakedefine01 FFMPEG_USE_SWRESAMPLE #cmakedefine01 APPIMAGE_BUILD +#cmakedefine ENABLE_GPU_FRAME +#ifdef ENABLE_GPU_FRAME +#define OPENSHOT_ENABLE_GPU_FRAME 1 +#endif #include diff --git a/src/Qt/VideoRenderer.cpp b/src/Qt/VideoRenderer.cpp index fdedcdb38..c5173c002 100644 --- a/src/Qt/VideoRenderer.cpp +++ b/src/Qt/VideoRenderer.cpp @@ -12,11 +12,15 @@ #include "VideoRenderer.h" #include +#include #include VideoRenderer::VideoRenderer(QObject *parent) : QObject(parent), override_widget(nullptr) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + qRegisterMetaType>(); +#endif } VideoRenderer::~VideoRenderer() @@ -26,6 +30,9 @@ VideoRenderer::~VideoRenderer() /// Override QWidget which needs to be painted void VideoRenderer::OverrideWidget(uintptr_t qwidget_address) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + frame_presenter.store(false); +#endif if (override_present_connection) QObject::disconnect(override_present_connection); @@ -34,6 +41,24 @@ void VideoRenderer::OverrideWidget(uintptr_t qwidget_address) if (!override_widget) return; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + QObject* presenter_object = override_widget; + auto presenter = dynamic_cast(presenter_object); + if (!presenter) { + presenter_object = override_widget->property("openshot_gpu_presenter").value(); + presenter = dynamic_cast(presenter_object); + } + if (presenter) { + override_present_connection = QObject::connect( + this, &VideoRenderer::presentFrame, presenter_object, + [presenter](std::shared_ptr frame) { + presenter->PresentFrame(frame); + }, Qt::QueuedConnection); + frame_presenter.store(true); + return; + } +#endif + override_present_connection = QObject::connect( this, &VideoRenderer::present, override_widget, @@ -55,3 +80,13 @@ void VideoRenderer::render(std::shared_ptr image) emit present(*image); } + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +void VideoRenderer::renderFrame(const std::shared_ptr& frame) +{ + if (frame_presenter.load()) + emit presentFrame(frame); + else + openshot::RendererBase::renderFrame(frame); +} +#endif diff --git a/src/Qt/VideoRenderer.h b/src/Qt/VideoRenderer.h index 1bdbdc893..a1db7ee34 100644 --- a/src/Qt/VideoRenderer.h +++ b/src/Qt/VideoRenderer.h @@ -19,11 +19,23 @@ #include #include #include +#include class QPainter; class QWidget; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +/// Implement on a native QWidget to receive retained frames on the GUI thread. +/// The presenter owns presentation synchronization and may retain the frame. +class FramePresenter { +public: + virtual ~FramePresenter() = default; + virtual void PresentFrame(const std::shared_ptr& frame) = 0; +}; +Q_DECLARE_METATYPE(std::shared_ptr) +#endif + class VideoRenderer : public QObject, public openshot::RendererBase { Q_OBJECT @@ -37,16 +49,25 @@ class VideoRenderer : public QObject, public openshot::RendererBase signals: void present(const QImage &image); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + void presentFrame(std::shared_ptr frame); +#endif protected: //void render(openshot::OSPixelFormat format, int width, int height, int bytesPerLine, unsigned char *data); void render(std::shared_ptr image); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + void renderFrame(const std::shared_ptr& frame) override; +#endif private slots: private: QWidget* override_widget; QMetaObject::Connection override_present_connection; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::atomic frame_presenter{false}; +#endif }; #endif //OPENSHOT_VIDEO_RENDERER_H diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp new file mode 100644 index 000000000..8a308b8c9 --- /dev/null +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -0,0 +1,345 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanOpenGLPresenter.h" +#include "VideoRenderer.h" +#include "../VulkanFrameImage.h" +#include "../VulkanCompositor.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +} + +namespace openshot { +namespace { +void check(VkResult result, const char* operation) { + if (result != VK_SUCCESS) throw std::runtime_error(operation); +} +template T glFunction(QOpenGLContext* context, const char* name) { + auto function = reinterpret_cast(context->getProcAddress(name)); + if (!function) throw std::runtime_error(name); + return function; +} +} +struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { + QPointer widget; + std::shared_ptr frame; + std::shared_ptr display_surface; + bool fallback_delivered = false; + uint64_t native_draws = 0; + uint64_t fallback_frames = 0; + explicit Impl(QWidget* value) : widget(value) { + value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); + } + ~Impl() override { + if (widget) widget->setProperty("openshot_gpu_presenter", QVariant()); + } + void PresentFrame(const std::shared_ptr& value) override { + frame = value; + display_surface.reset(); + fallback_delivered = false; + if (frame && !frame->GetDeferredImage()) fallback(); + if (widget) { + if (frame && frame->GetDeferredImage() && widget->metaObject()->indexOfMethod("gpu_presented(int)") >= 0) + QMetaObject::invokeMethod(widget, "gpu_presented", Qt::DirectConnection, + Q_ARG(int, static_cast(frame->number))); + widget->update(); + } + } + void fallback() { + if (!fallback_delivered && frame && widget) { + fallback_delivered = true; + ++fallback_frames; + display_surface.reset(); + const QImage image = *frame->GetImage(); + QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + } + } + bool draw(int x, int y, int width, int height) { + auto lastStage = std::chrono::steady_clock::now(); + const bool trace = qEnvironmentVariableIsSet("OPENSHOT_TRACE_PRESENTATION"); + auto stage = [&](const char* name) { + if (!trace) return; + auto now = std::chrono::steady_clock::now(); + qWarning() << "GPU presentation" << name << std::chrono::duration(now-lastStage).count(); + lastStage = now; + }; + auto context = QOpenGLContext::currentContext(); + if (!context || !frame || !widget || fallback_delivered) return false; + auto authoritative = frame->GetDeferredImage(); + if (!authoritative) return false; + auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(authoritative); + if (!surface) return false; + auto inputContext = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); + if (inputContext->sw_format != AV_PIX_FMT_RGBA) { + surface = std::dynamic_pointer_cast(RenderVulkanImage( + surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); + if (!surface) return false; + } + display_surface = surface; + stage("surface"); + const AVFrame* source = surface->GetFrame(); + auto fc = reinterpret_cast(source->hw_frames_ctx->data); + auto vfc = static_cast(fc->hwctx); + auto dc = fc->device_ctx; + auto vk = static_cast(dc->hwctx); + auto image = reinterpret_cast(source->data[0]); + // Multi-plane decode surfaces must first go through the RGBA compositor. + if (fc->sw_format != AV_PIX_FMT_RGBA || vfc->format[0] != VK_FORMAT_R8G8B8A8_UNORM || image->img[1]) + return false; + if (!context->hasExtension("GL_EXT_memory_object_fd") || !context->hasExtension("GL_EXT_semaphore_fd")) + return false; + auto gl = context->functions(); + auto extra = context->extraFunctions(); + auto getUuid = glFunction(context, "glGetUnsignedBytei_vEXT"); + VkPhysicalDeviceIDProperties identity{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity}; + vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); + GLint deviceCount = 0; + gl->glGetIntegerv(GL_NUM_DEVICE_UUIDS_EXT, &deviceCount); + bool sameDevice = false; + for (int i = 0; i < deviceCount; ++i) { + GLubyte uuid[GL_UUID_SIZE_EXT]; + getUuid(GL_DEVICE_UUID_EXT, i, uuid); + sameDevice = sameDevice || std::memcmp(uuid, identity.deviceUUID, VK_UUID_SIZE) == 0; + } + if (!sameDevice) return false; + auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); + auto deleteMemory = glFunction(context, "glDeleteMemoryObjectsEXT"); + auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); + auto importMemory = glFunction(context, "glImportMemoryFdEXT"); + auto textureMemory = glFunction(context, "glTexStorageMem2DEXT"); + auto createSemaphore = glFunction(context, "glGenSemaphoresEXT"); + auto deleteSemaphore = glFunction(context, "glDeleteSemaphoresEXT"); + auto importSemaphore = glFunction(context, "glImportSemaphoreFdEXT"); + auto waitSemaphore = glFunction(context, "glWaitSemaphoreEXT"); + auto signalSemaphore = glFunction(context, "glSignalSemaphoreEXT"); + auto getMemoryFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryFdKHR")); + auto getSemaphoreFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreFdKHR")); + if (!getMemoryFd || !getSemaphoreFd || vk->queue_family_comp_index < 0) return false; + // Construct all fallible GL resources before transferring image ownership. + QOpenGLShaderProgram program; + if (!program.addShaderFromSourceCode(QOpenGLShader::Vertex, + "attribute vec2 vertex; varying vec2 uv; void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}") || + !program.addShaderFromSourceCode(QOpenGLShader::Fragment, + "uniform sampler2D video; varying vec2 uv; void main(){gl_FragColor=texture2D(video,uv);}") || !program.link()) return false; + stage("shader"); + VkDevice device = vk->act_dev; + GLuint memory = 0, texture = 0, semaphores[2] = {0, 0}, buffer = 0, vertexArray = 0; + GLint previousVertexArray = 0; + gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); + VkSemaphore handoff[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkCommandPool pool = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + int memoryFd = -1, semaphoreFd = -1; + auto cleanup = [&]() { + if (vertexArray) { + extra->glBindVertexArray(previousVertexArray); + extra->glDeleteVertexArrays(1, &vertexArray); + } + if (buffer) gl->glDeleteBuffers(1, &buffer); + if (texture) gl->glDeleteTextures(1, &texture); + stage("delete texture"); + if (memory) deleteMemory(1, &memory); + stage("delete memory"); + deleteSemaphore(2, semaphores); + stage("delete GL semaphore"); + if (fence) vkDestroyFence(device, fence, nullptr); + if (pool) vkDestroyCommandPool(device, pool, nullptr); + for (auto semaphore : handoff) + if (semaphore) vkDestroySemaphore(device, semaphore, nullptr); + stage("delete Vulkan resources"); + if (memoryFd >= 0) close(memoryFd); + if (semaphoreFd >= 0) close(semaphoreFd); + }; + bool locked = false; + try { + VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; + memoryInfo.memory = image->mem[0]; + memoryInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + check(getMemoryFd(device, &memoryInfo, &memoryFd), "Export Vulkan image memory"); + createMemory(1, &memory); + VkMemoryDedicatedRequirements dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 requirements{VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated}; + VkImageMemoryRequirementsInfo2 requirementInfo{VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2}; + requirementInfo.image = image->img[0]; + vkGetImageMemoryRequirements2(device, &requirementInfo, &requirements); + GLint useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; + memoryParameter(memory, GL_DEDICATED_MEMORY_OBJECT_EXT, &useDedicated); + importMemory(memory, image->size[0], GL_HANDLE_TYPE_OPAQUE_FD_EXT, memoryFd); + memoryFd = -1; // GL consumes imported descriptors. + stage("import memory"); + gl->glGenTextures(1, &texture); + gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + textureMemory(GL_TEXTURE_2D, 1, GL_RGBA8, source->width, source->height, memory, image->offset[0]); + stage("texture storage"); + VkExportSemaphoreCreateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; + exportInfo.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; + VkSemaphoreCreateInfo semaphoreInfo{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &exportInfo}; + // Separate directions: reusing one binary semaphore lets Vulkan's + // acquire consume its own release signal before GL can wait on it. + createSemaphore(2, semaphores); + for (int direction = 0; direction < 2; ++direction) { + check(vkCreateSemaphore(device, &semaphoreInfo, nullptr, &handoff[direction]), "Create presentation semaphore"); + VkSemaphoreGetFdInfoKHR fdInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; + fdInfo.semaphore = handoff[direction]; + fdInfo.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; + check(getSemaphoreFd(device, &fdInfo, &semaphoreFd), "Export presentation semaphore"); + importSemaphore(semaphores[direction], GL_HANDLE_TYPE_OPAQUE_FD_EXT, semaphoreFd); + semaphoreFd = -1; + } + if (gl->glGetError() != GL_NO_ERROR) throw std::runtime_error("OpenGL external memory import failed"); + const uint32_t family = vk->queue_family_comp_index; + VkQueue queue; + vkGetDeviceQueue(device, family, 0, &queue); + VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + poolInfo.queueFamilyIndex = family; + check(vkCreateCommandPool(device, &poolInfo, nullptr, &pool), "Create presentation command pool"); + VkCommandBuffer commands[2]; + VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocation.commandPool = pool; + allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocation.commandBufferCount = 2; + check(vkAllocateCommandBuffers(device, &allocation, commands), "Allocate presentation commands"); + VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + check(vkCreateFence(device, &fenceInfo, nullptr, &fence), "Create presentation fence"); + const float vertices[] = {-1,-1, 1,-1, -1,1, 1,1}; + extra->glGenVertexArrays(1, &vertexArray); + extra->glBindVertexArray(vertexArray); + gl->glGenBuffers(1, &buffer); + gl->glBindBuffer(GL_ARRAY_BUFFER, buffer); + gl->glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + program.bind(); + program.setUniformValue("video", 0); + const int vertex = program.attributeLocation("vertex"); + program.enableAttributeArray(vertex); + program.setAttributeBuffer(vertex, GL_FLOAT, 0, 2); + vfc->lock_frame(fc, image); + locked = true; + stage("frame lock"); + const auto oldLayout = image->layout[0]; + const auto oldOwner = image->queue_family[0]; + for (int i = 0; i < 2; ++i) { + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + check(vkBeginCommandBuffer(commands[i], &begin), "Begin presentation commands"); + VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + barrier.srcAccessMask = i ? 0 : image->access[0]; + barrier.dstAccessMask = i ? VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT : 0; + barrier.oldLayout = i ? VK_IMAGE_LAYOUT_GENERAL : oldLayout; + barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; + barrier.srcQueueFamilyIndex = i ? VK_QUEUE_FAMILY_EXTERNAL : oldOwner; + barrier.dstQueueFamilyIndex = i ? family : VK_QUEUE_FAMILY_EXTERNAL; + barrier.image = image->img[0]; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vkCmdPipelineBarrier(commands[i], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + check(vkEndCommandBuffer(commands[i]), "End presentation commands"); + } + auto submit = [&](int index, VkSemaphore wait, uint64_t waitValue, VkSemaphore signal, uint64_t signalValue) { + VkTimelineSemaphoreSubmitInfo timeline{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; + timeline.waitSemaphoreValueCount = timeline.signalSemaphoreValueCount = 1; + timeline.pWaitSemaphoreValues = &waitValue; + timeline.pSignalSemaphoreValues = &signalValue; + VkPipelineStageFlags stage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkSubmitInfo info{VK_STRUCTURE_TYPE_SUBMIT_INFO, &timeline}; + info.waitSemaphoreCount = info.signalSemaphoreCount = info.commandBufferCount = 1; + info.pWaitSemaphores = &wait; + info.pWaitDstStageMask = &stage; + info.pSignalSemaphores = &signal; + info.pCommandBuffers = &commands[index]; + vk->lock_queue(dc, family, 0); + auto result = vkQueueSubmit(queue, 1, &info, index ? fence : VK_NULL_HANDLE); + vk->unlock_queue(dc, family, 0); + check(result, "Submit presentation transfer"); + }; + submit(0, image->sem[0], image->sem_value[0], handoff[0], 0); + stage("release submit"); + const GLenum layout = GL_LAYOUT_GENERAL_EXT; + waitSemaphore(semaphores[0], 0, nullptr, 1, &texture, &layout); + GLint viewport[4]; + gl->glGetIntegerv(GL_VIEWPORT, viewport); + gl->glViewport(x, y, width, height); + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glDisable(GL_DEPTH_TEST); + gl->glDisable(GL_SCISSOR_TEST); + gl->glDisable(GL_STENCIL_TEST); + gl->glDisable(GL_BLEND); + gl->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + gl->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + signalSemaphore(semaphores[1], 0, nullptr, 1, &texture, &layout); + gl->glFlush(); + stage("GL draw flush"); + submit(1, handoff[1], 0, image->sem[0], image->sem_value[0] + 1); + image->sem_value[0]++; + image->layout[0] = VK_IMAGE_LAYOUT_GENERAL; + image->access[0] = VK_ACCESS_MEMORY_READ_BIT; + image->queue_family[0] = family; + vfc->unlock_frame(fc, image); + locked = false; + // Synchronous prototype: no resource reuse before GL and Vulkan finish. + check(vkWaitForFences(device, 1, &fence, VK_TRUE, UINT64_MAX), "Wait for presentation transfer"); + stage("acquire fence"); + gl->glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); + program.disableAttributeArray(vertex); + program.release(); + gl->glBindBuffer(GL_ARRAY_BUFFER, 0); + // Retire GL's imported-object command references as well as the + // Vulkan queue work before destroying the cross-API resources. + gl->glFinish(); + stage("GL finish"); + cleanup(); + stage("cleanup"); + return true; + } catch (...) { + if (locked) vfc->unlock_frame(fc, image); + cleanup(); + throw; + } + } +}; +VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) + : impl(new Impl(reinterpret_cast(widget))) {} +VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; +bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { + try { + if (impl->draw(x, y, width, height)) { ++impl->native_draws; return true; } + } catch (const std::exception& error) { + qWarning() << "Vulkan presentation fallback:" << error.what(); + } + impl->fallback(); + return false; +} +int VulkanOpenGLPresenter::Width() const { return impl->frame ? impl->frame->GetWidth() : 0; } +int VulkanOpenGLPresenter::Height() const { return impl->frame ? impl->frame->GetHeight() : 0; } +void VulkanOpenGLPresenter::Clear() { + impl->frame.reset(); + impl->display_surface.reset(); +} +uint64_t VulkanOpenGLPresenter::NativeDraws() const { return impl->native_draws; } +uint64_t VulkanOpenGLPresenter::FallbackFrames() const { return impl->fallback_frames; } +void VulkanOpenGLPresenter::Present(std::shared_ptr frame) { + if (QThread::currentThread() == impl->thread()) { + impl->PresentFrame(frame); + } else { + QMetaObject::invokeMethod(impl.get(), [receiver=impl.get(), frame]() { + receiver->PresentFrame(frame); + }, Qt::QueuedConnection); + } +} +} diff --git a/src/Qt/VulkanOpenGLPresenter.h b/src/Qt/VulkanOpenGLPresenter.h new file mode 100644 index 000000000..b22543046 --- /dev/null +++ b/src/Qt/VulkanOpenGLPresenter.h @@ -0,0 +1,26 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_OPENGL_PRESENTER_H +#define OPENSHOT_VULKAN_OPENGL_PRESENTER_H +#include +#include +namespace openshot { +class Frame; +/// GUI-thread adapter for a QOpenGLWidget. Draw requires its current GL context. +class VulkanOpenGLPresenter { +public: + explicit VulkanOpenGLPresenter(uintptr_t widget); + ~VulkanOpenGLPresenter(); + bool Draw(int x, int y, int width, int height); + int Width() const; + int Height() const; + void Clear(); + uint64_t NativeDraws() const; + uint64_t FallbackFrames() const; + void Present(std::shared_ptr frame); +private: + struct Impl; + std::unique_ptr impl; +}; +} +#endif diff --git a/src/RendererBase.cpp b/src/RendererBase.cpp index c172600a2..2b7250f38 100644 --- a/src/RendererBase.cpp +++ b/src/RendererBase.cpp @@ -24,5 +24,16 @@ RendererBase::~RendererBase() void RendererBase::paint(const std::shared_ptr & frame) { if (frame) +#ifdef OPENSHOT_ENABLE_GPU_FRAME + this->renderFrame(frame); +#else this->render(frame->GetImage()); +#endif } + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +void RendererBase::renderFrame(const std::shared_ptr& frame) +{ + render(frame->GetImage()); +} +#endif diff --git a/src/RendererBase.h b/src/RendererBase.h index 49ed3cc31..b060e24b5 100644 --- a/src/RendererBase.h +++ b/src/RendererBase.h @@ -43,6 +43,10 @@ namespace openshot virtual ~RendererBase(); virtual void render(std::shared_ptr image) = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + /// Override to consume retained surfaces before CPU materialization. + virtual void renderFrame(const std::shared_ptr& frame); +#endif }; } diff --git a/src/Timeline.cpp b/src/Timeline.cpp index c34097e24..51271af47 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -76,6 +76,9 @@ Timeline::Timeline(int width, int height, Fraction fps, int sample_rate, int cha final_cache = new CacheMemory(); const int cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4); final_cache->SetMaxBytesFromInfo(cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache->SetMaxBytes(std::min(final_cache->GetMaxBytes(), 256LL * 1024 * 1024)); +#endif } // Delegating constructor that copies parameters from a provided ReaderInfo @@ -209,6 +212,9 @@ Timeline::Timeline(const std::string& projectPath, bool convert_absolute_paths) final_cache = new CacheMemory(); const int cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 4); final_cache->SetMaxBytesFromInfo(cache_frames, info.width, info.height, info.sample_rate, info.channels); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + final_cache->SetMaxBytes(std::min(final_cache->GetMaxBytes(), 256LL * 1024 * 1024)); +#endif } Timeline::~Timeline() { diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp new file mode 100644 index 000000000..88b09057a --- /dev/null +++ b/src/VulkanCompositor.cpp @@ -0,0 +1,337 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanCompositor.h" +#include "VulkanEncodeConverter.h" +#include "VulkanFrameImage.h" +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +} +namespace openshot { +namespace { +void VkCheck(VkResult r, const char* operation) { + if(r != VK_SUCCESS) throw std::runtime_error(std::string(operation)+": Vulkan error "+std::to_string(r)); +} +void AvCheck(int r, const char* operation) { + if(r < 0) throw std::runtime_error(std::string(operation)+": FFmpeg error "+std::to_string(r)); +} +struct FrameDelete { void operator()(AVFrame* f) const { av_frame_free(&f); } }; +using FramePtr=std::unique_ptr; +struct Parameters { + float matrix_x[4], matrix_y[4], coeffs[4], extra[4], clear[4]; + int sizes[4], modes[4]; +}; +struct LockedFrame { + AVHWFramesContext* context; + AVVulkanFramesContext* vkcontext; + AVVkFrame* frame; + explicit LockedFrame(const AVFrame* f) { + context=reinterpret_cast(f->hw_frames_ctx->data); + vkcontext=reinterpret_cast(context->hwctx); + frame=reinterpret_cast(f->data[0]); + vkcontext->lock_frame(context,frame); + } + ~LockedFrame(){vkcontext->unlock_frame(context,frame);} +}; +class Pipeline { + AVBufferRef* frames=nullptr; + AVHWDeviceContext* device_context=nullptr; + AVVulkanDeviceContext* vkcontext=nullptr; + VkDevice device=VK_NULL_HANDLE; + VkQueue queue=VK_NULL_HANDLE; + int queue_family=-1; + VkCommandPool commands=VK_NULL_HANDLE; + VkDescriptorSetLayout descriptors=VK_NULL_HANDLE; + VkDescriptorPool pool=VK_NULL_HANDLE; + VkPipelineLayout layout=VK_NULL_HANDLE; + VkPipeline pipeline=VK_NULL_HANDLE; + VkSampler sampler=VK_NULL_HANDLE; + VkDescriptorSet set=VK_NULL_HANDLE; + int width,height; + bool encode_nv12; +public: + Pipeline(AVBufferRef* device_ref,int w,int h,bool encode=false):width(w),height(h),encode_nv12(encode) { + device_context=reinterpret_cast(device_ref->data); + vkcontext=reinterpret_cast(device_context->hwctx); + device=vkcontext->act_dev; + queue_family=vkcontext->queue_family_comp_index; + if(queue_family<0) throw std::runtime_error("No Vulkan compute queue"); + try { + frames=av_hwframe_ctx_alloc(device_ref); + if(!frames) throw std::bad_alloc(); + auto* fc=reinterpret_cast(frames->data); + fc->format=AV_PIX_FMT_VULKAN; fc->sw_format=encode_nv12?AV_PIX_FMT_NV12:AV_PIX_FMT_RGBA; + fc->width=w; fc->height=h; + auto* vc=reinterpret_cast(fc->hwctx); + if(encode_nv12) vc->flags=AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + vc->usage=static_cast(VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT); + AvCheck(av_hwframe_ctx_init(frames),"Initialize Vulkan compositor output"); + vkGetDeviceQueue(device,queue_family,0,&queue); + VkCommandPoolCreateInfo ci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + ci.queueFamilyIndex=queue_family; ci.flags=VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + VkCheck(vkCreateCommandPool(device,&ci,nullptr,&commands),"Create command pool"); + std::array bindings{}; + for(int i=0;i<5;++i) { + bindings[i].binding=i; bindings[i].descriptorCount=1; + bindings[i].descriptorType=(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + bindings[i].stageFlags=VK_SHADER_STAGE_COMPUTE_BIT; + } + VkDescriptorSetLayoutCreateInfo di{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + di.bindingCount=5; di.pBindings=bindings.data(); + VkCheck(vkCreateDescriptorSetLayout(device,&di,nullptr,&descriptors),"Create descriptors"); + VkPushConstantRange range{VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(Parameters)}; + VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + li.setLayoutCount=1; li.pSetLayouts=&descriptors; li.pushConstantRangeCount=1; li.pPushConstantRanges=⦥ + VkCheck(vkCreatePipelineLayout(device,&li,nullptr,&layout),"Create pipeline layout"); + std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,4},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2}}}; + VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + pi.maxSets=1; pi.poolSizeCount=2; pi.pPoolSizes=sizes.data(); + VkCheck(vkCreateDescriptorPool(device,&pi,nullptr,&pool),"Create descriptor pool"); + VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + ai.descriptorPool=pool; ai.descriptorSetCount=1; ai.pSetLayouts=&descriptors; + VkCheck(vkAllocateDescriptorSets(device,&ai,&set),"Allocate descriptors"); + VkSamplerCreateInfo si{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + si.magFilter=si.minFilter=VK_FILTER_LINEAR; + si.addressModeU=si.addressModeV=si.addressModeW=VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + VkCheck(vkCreateSampler(device,&si,nullptr,&sampler),"Create sampler"); + std::ifstream shader(encode_nv12?OPENSHOT_VULKAN_ENCODE_SHADER_PATH:OPENSHOT_VULKAN_AFFINE_SHADER_PATH,std::ios::binary|std::ios::ate); + if(!shader) throw std::runtime_error("Vulkan affine shader unavailable"); + const auto bytes=shader.tellg(); + if(bytes<=0 || bytes%4) throw std::runtime_error("Invalid Vulkan affine shader"); + std::vector code(static_cast(bytes)/4); + shader.seekg(0); shader.read(reinterpret_cast(code.data()),bytes); + VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + mi.codeSize=code.size()*4; mi.pCode=code.data(); + VkShaderModule module; + VkCheck(vkCreateShaderModule(device,&mi,nullptr,&module),"Create shader module"); + VkComputePipelineCreateInfo pci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pci.layout=layout; pci.stage.sType=VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pci.stage.stage=VK_SHADER_STAGE_COMPUTE_BIT; pci.stage.module=module; pci.stage.pName="main"; + const auto status=vkCreateComputePipelines(device,VK_NULL_HANDLE,1,&pci,nullptr,&pipeline); + vkDestroyShaderModule(device,module,nullptr); + VkCheck(status,"Create affine pipeline"); + }catch(...){Destroy();throw;} + } + ~Pipeline(){Destroy();} + void Destroy(){ + if(pipeline) vkDestroyPipeline(device,pipeline,nullptr); + if(sampler) vkDestroySampler(device,sampler,nullptr); + if(pool) vkDestroyDescriptorPool(device,pool,nullptr); + if(layout) vkDestroyPipelineLayout(device,layout,nullptr); + if(descriptors) vkDestroyDescriptorSetLayout(device,descriptors,nullptr); + if(commands) vkDestroyCommandPool(device,commands,nullptr); + av_buffer_unref(&frames); + } + VkImageView View(VkImage image,VkFormat format,VkImageAspectFlags aspect){ + VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + vi.image=image; vi.viewType=VK_IMAGE_VIEW_TYPE_2D; vi.format=format; + vi.subresourceRange.aspectMask=aspect; vi.subresourceRange.levelCount=1; vi.subresourceRange.layerCount=1; + VkImageView view; + VkCheck(vkCreateImageView(device,&vi,nullptr,&view),"Create affine image view"); + return view; + } + std::shared_ptr Render(const VulkanFrameImage& input,const VulkanFrameImage* background, + const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false) { + const auto* src=input.GetFrame(); + const auto* src_fc=reinterpret_cast(src->hw_frames_ctx->data); + if(src_fc->device_ctx!=device_context) throw std::runtime_error("Vulkan source device mismatch"); + const AVFrame* back=background?background->GetFrame():nullptr; + if(back){ + const auto* bc=reinterpret_cast(back->hw_frames_ctx->data); + if(bc->device_ctx!=device_context || bc->sw_format!=AV_PIX_FMT_RGBA || back->width!=width || back->height!=height) + throw std::runtime_error("Vulkan background must be matching RGBA surface"); + } + FramePtr output(av_frame_alloc()); + if(!output) throw std::bad_alloc(); + AvCheck(av_hwframe_get_buffer(frames,output.get(),0),"Allocate compositor output"); + AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); + output->colorspace=encode_nv12?AVCOL_SPC_SMPTE170M:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; + if(encode_nv12 && src_fc->sw_format!=AV_PIX_FMT_RGBA) throw std::runtime_error("Encoder conversion requires RGBA"); + Parameters p{}; + bool invertible=false; + const auto inverse=matrix.inverted(&invertible); + if(!invertible) opacity=0; + p.matrix_x[0]=inverse.m11();p.matrix_x[1]=inverse.m21();p.matrix_x[2]=inverse.dx(); + p.matrix_y[0]=inverse.m12();p.matrix_y[1]=inverse.m22();p.matrix_y[2]=inverse.dy(); + p.sizes[0]=width;p.sizes[1]=height;p.sizes[2]=src->width;p.sizes[3]=src->height; + p.extra[3]=std::clamp(opacity,0.0,1.0); + p.clear[3]=clear.alphaF();p.clear[0]=clear.redF()*p.clear[3];p.clear[1]=clear.greenF()*p.clear[3];p.clear[2]=clear.blueF()*p.clear[3]; + p.modes[1]=back?1:0; + p.modes[2]=high_quality?1:0; + int plane_count=0; + if(src_fc->sw_format==AV_PIX_FMT_RGBA){p.modes[0]=input.IsPremultiplied()?2:3;plane_count=1;} + else if(src_fc->sw_format==AV_PIX_FMT_NV12){p.modes[0]=1;plane_count=1;} + else if(src_fc->sw_format==AV_PIX_FMT_YUV420P){p.modes[0]=0;plane_count=3;} + else throw std::runtime_error("Vulkan affine renderer supports RGBA, NV12 and YUV420P"); + double kr=0.299,kb=0.114; + if(src->colorspace==AVCOL_SPC_BT709){kr=0.2126;kb=0.0722;} + else if(src->colorspace==AVCOL_SPC_BT2020_NCL){kr=0.2627;kb=0.0593;} + else if(src->colorspace==AVCOL_SPC_SMPTE240M){kr=0.212;kb=0.087;} + const bool full=src->color_range==AVCOL_RANGE_JPEG; + const double cscale=full?1.0:255.0/224.0; + p.coeffs[0]=full?1.0:255.0/219.0;p.extra[1]=full?0.0:16.0/255.0;p.extra[2]=128.0/255.0; + p.coeffs[1]=2*(1-kr)*cscale;p.coeffs[2]=-2*kb*(1-kb)/(1-kr-kb)*cscale; + p.coeffs[3]=-2*kr*(1-kr)/(1-kr-kb)*cscale;p.extra[0]=2*(1-kb)*cscale; + // Lock in stable address order, avoiding deadlocks when frames are shared. + std::vector all{src,output.get()}; + if(back && back->data[0]!=src->data[0]) all.push_back(back); + std::sort(all.begin(),all.end(),[](const AVFrame* a,const AVFrame* b){return std::less()(a->data[0],b->data[0]);}); + std::vector> locks; + for(auto* f:all) locks.emplace_back(new LockedFrame(f)); + auto* sf=reinterpret_cast(src->data[0]); + auto* of=reinterpret_cast(output->data[0]); + const bool separate_nv12 = p.modes[0]==1 && sf->img[1]!=VK_NULL_HANDLE && sf->img[1]!=sf->img[0]; + if(separate_nv12) plane_count=2; + const auto* svc=reinterpret_cast(src_fc->hwctx); + std::vector owned_views; + VkCommandBuffer cmd=VK_NULL_HANDLE; + VkFence fence=VK_NULL_HANDLE; + bool submitted=false; + try { + std::array views{}; + auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ + auto view=View(image,format,aspect);owned_views.push_back(view);return view; + }; + if(p.modes[0]==1){ + views[0]=make_view(sf->img[0],VK_FORMAT_R8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); + views[1]=views[2]=make_view(sf->img[separate_nv12?1:0],VK_FORMAT_R8G8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); + }else if(p.modes[0]>=2){ + views[0]=views[1]=views[2]=make_view(sf->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + }else{ + for(int i=0;i<3;++i) views[i]=make_view(sf->img[i],svc->format[i],VK_IMAGE_ASPECT_COLOR_BIT); + } + views[3]=back?make_view(reinterpret_cast(back->data[0])->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT):views[0]; + if(encode_nv12){ + if(!of->img[1] || of->img[1]==of->img[0]) throw std::runtime_error("NV12 encoder output requires separate plane images"); + views[3]=make_view(of->img[0],VK_FORMAT_R8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + views[4]=make_view(of->img[1],VK_FORMAT_R8G8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + }else views[4]=make_view(of->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + std::array images{}; + std::array writes{}; + for(int i=0;i<5;++i){ + images[i].sampler=(i==4 || (encode_nv12 && i==3))?VK_NULL_HANDLE:sampler;images[i].imageView=views[i];images[i].imageLayout=VK_IMAGE_LAYOUT_GENERAL; + writes[i].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;writes[i].dstSet=set;writes[i].dstBinding=i;writes[i].descriptorCount=1; + writes[i].descriptorType=(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;writes[i].pImageInfo=&images[i]; + } + vkUpdateDescriptorSets(device,5,writes.data(),0,nullptr); + VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ai.commandPool=commands;ai.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY;ai.commandBufferCount=1; + VkCheck(vkAllocateCommandBuffers(device,&ai,&cmd),"Allocate affine commands"); + VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};bi.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + VkCheck(vkBeginCommandBuffer(cmd,&bi),"Begin affine commands"); + std::vector semaphores; + std::vector waits,signal_values; + std::vector> changed; + std::vector barriers; + for(auto* f:all){ + auto* vf=reinterpret_cast(f->data[0]); + const int count=f==src?plane_count:(f==output.get() && encode_nv12?2:1); + for(int i=0;i(vf->access[i]); + barrier.dstAccessMask=f==output.get()?VK_ACCESS_SHADER_WRITE_BIT:VK_ACCESS_SHADER_READ_BIT; + barrier.oldLayout=vf->layout[i];barrier.newLayout=VK_IMAGE_LAYOUT_GENERAL; + barrier.srcQueueFamilyIndex=vf->queue_family[i]; + barrier.dstQueueFamilyIndex=vf->queue_family[i]==VK_QUEUE_FAMILY_IGNORED?VK_QUEUE_FAMILY_IGNORED:uint32_t(queue_family); + barrier.image=vf->img[i];barrier.subresourceRange.aspectMask=VK_IMAGE_ASPECT_COLOR_BIT; + barrier.subresourceRange.levelCount=1;barrier.subresourceRange.layerCount=1; + barriers.push_back(barrier);changed.emplace_back(vf,i); + if(std::find(semaphores.begin(),semaphores.end(),vf->sem[i])==semaphores.end()){ + semaphores.push_back(vf->sem[i]);waits.push_back(vf->sem_value[i]);signal_values.push_back(vf->sem_value[i]+1); + } + } + } + vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,0,0,nullptr,0,nullptr,barriers.size(),barriers.data()); + vkCmdBindPipeline(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,pipeline); + vkCmdBindDescriptorSets(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,layout,0,1,&set,0,nullptr); + vkCmdPushConstants(cmd,layout,VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(p),&p); + vkCmdDispatch(cmd,(width+15)/16,(height+15)/16,1); + VkCheck(vkEndCommandBuffer(cmd),"End affine commands"); + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};VkCheck(vkCreateFence(device,&fi,nullptr,&fence),"Create affine fence"); + VkTimelineSemaphoreSubmitInfo ti{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; + ti.waitSemaphoreValueCount=waits.size();ti.pWaitSemaphoreValues=waits.data();ti.signalSemaphoreValueCount=signal_values.size();ti.pSignalSemaphoreValues=signal_values.data(); + std::vector stages(waits.size(),VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo si{VK_STRUCTURE_TYPE_SUBMIT_INFO};si.pNext=&ti; + si.waitSemaphoreCount=semaphores.size();si.pWaitSemaphores=semaphores.data();si.pWaitDstStageMask=stages.data(); + si.signalSemaphoreCount=semaphores.size();si.pSignalSemaphores=semaphores.data();si.commandBufferCount=1;si.pCommandBuffers=&cmd; + vkcontext->lock_queue(device_context,queue_family,0); + auto result=vkQueueSubmit(queue,1,&si,fence); + vkcontext->unlock_queue(device_context,queue_family,0); + VkCheck(result,"Submit affine renderer");submitted=true; + for(size_t changed_index=0;changed_indexlayout[item.second]=VK_IMAGE_LAYOUT_GENERAL; + item.first->access[item.second]=item.first==of?VK_ACCESS_SHADER_WRITE_BIT:VK_ACCESS_SHADER_READ_BIT; + item.first->queue_family[item.second]=barriers[changed_index].dstQueueFamilyIndex; + item.first->sem_value[item.second]++; + } + // Publish semaphore values before releasing frame locks. Never hold a queue lock while waiting. + locks.clear(); + VkCheck(vkWaitForFences(device,1,&fence,VK_TRUE,UINT64_MAX),"Wait affine renderer"); + submitted=false; + vkDestroyFence(device,fence,nullptr);fence=VK_NULL_HANDLE; + vkFreeCommandBuffers(device,commands,1,&cmd);cmd=VK_NULL_HANDLE; + for(auto view:owned_views) vkDestroyImageView(device,view,nullptr); + owned_views.clear(); // A later allocation failure must not destroy views twice. + return std::make_shared(output.get(),!encode_nv12); + }catch(...){ + if(submitted && fence) vkWaitForFences(device,1,&fence,VK_TRUE,UINT64_MAX); + if(fence) vkDestroyFence(device,fence,nullptr); + if(cmd) vkFreeCommandBuffers(device,commands,1,&cmd); + for(auto view:owned_views) vkDestroyImageView(device,view,nullptr); + throw; + } + } +}; +} +std::shared_ptr RenderVulkanImage(std::shared_ptr source, + std::shared_ptr background,int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality){ + auto src=std::dynamic_pointer_cast(source); + auto back=std::dynamic_pointer_cast(background); + if(!src || (background && !back)) throw std::invalid_argument("Vulkan compositor requires Vulkan surfaces"); + if(width<=0 || height<=0) throw std::invalid_argument("Invalid compositor dimensions"); + auto* fc=reinterpret_cast(src->GetFrame()->hw_frames_ctx->data); + using Key=std::tuple; + // Reader scaling and timeline transforms alternate output dimensions. Keep a + // bounded LRU so those operations do not recompile a pipeline on every frame. + struct CachedPipeline { std::unique_ptr pipeline; uint64_t used; }; + thread_local std::map cache; + thread_local uint64_t clock=0; + const Key requested{fc->device_ctx,width,height}; + auto found=cache.find(requested); + if(found==cache.end()){ + if(cache.size()>=8){ + auto oldest=std::min_element(cache.begin(),cache.end(),[](const auto& a,const auto& b){return a.second.used(fc->device_ref,width,height),0}).first; + } + found->second.used=++clock; + auto& pipeline=found->second.pipeline; + return pipeline->Render(*src,back.get(),matrix,opacity,clear,high_quality); +} +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source) { + auto src=std::dynamic_pointer_cast(source); + if(!src) throw std::invalid_argument("NV12 conversion requires a Vulkan surface"); + const auto* frame=src->GetFrame(); + if(frame->width%2 || frame->height%2) throw std::invalid_argument("NV12 export requires even dimensions"); + auto* fc=reinterpret_cast(frame->hw_frames_ctx->data); + using Key=std::tuple; + thread_local std::unique_ptr pipeline; + thread_local Key key{}; + Key requested{fc->device_ctx,frame->width,frame->height}; + if(!pipeline || key!=requested){ + pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true); + key=requested; + } + return pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black); +} + +} diff --git a/src/VulkanCompositor.h b/src/VulkanCompositor.h new file mode 100644 index 000000000..b4988c63c --- /dev/null +++ b/src/VulkanCompositor.h @@ -0,0 +1,18 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_COMPOSITOR_H +#define OPENSHOT_VULKAN_COMPOSITOR_H +#include "FrameImage.h" +#include +#include +#include +namespace openshot { +/// Affine transform and source-over compositing into an independently owned GPU frame. +/// Inputs must be Vulkan surfaces from the same device; null background uses clear. +/// high_quality selects antialiased bicubic reader resampling; default matches Qt bilinear transforms. +std::shared_ptr RenderVulkanImage( + std::shared_ptr source, std::shared_ptr background, + int width, int height, const QTransform& source_to_output, double opacity, + const QColor& clear = Qt::transparent, bool high_quality = false); +} +#endif diff --git a/src/VulkanEncodeConverter.h b/src/VulkanEncodeConverter.h new file mode 100644 index 000000000..232464a59 --- /dev/null +++ b/src/VulkanEncodeConverter.h @@ -0,0 +1,11 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_ENCODE_CONVERTER_H +#define OPENSHOT_VULKAN_ENCODE_CONVERTER_H +#include "FrameImage.h" +#include +namespace openshot { +/// Convert an RGBA Vulkan surface to encoder-ready limited-range NV12 on the GPU. +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source); +} +#endif diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp new file mode 100644 index 000000000..4d3b71fd5 --- /dev/null +++ b/src/VulkanFrameImage.cpp @@ -0,0 +1,118 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanFrameImage.h" +#include "VulkanCompositor.h" +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +#include +#include +#include +#include +} +namespace openshot { +namespace { +void Check(int result, const char* operation) { + if (result >= 0) return; + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, error, sizeof(error)); + throw std::runtime_error(std::string(operation) + ": " + error); +} +struct FrameDeleter { void operator()(AVFrame* f) const { av_frame_free(&f); } }; +using OwnedFrame = std::unique_ptr; +OwnedFrame AllocateFrame() { + OwnedFrame f(av_frame_alloc()); + if (!f) throw std::bad_alloc(); + return f; +} +void Validate(const AVFrame* frame) { + if (!frame || frame->format != AV_PIX_FMT_VULKAN || !frame->hw_frames_ctx || + !frame->data[0] || frame->width <= 0 || frame->height <= 0) + throw std::invalid_argument("Expected an FFmpeg Vulkan hardware frame"); +} +} +struct VulkanFrameImage::Impl { + OwnedFrame frame; + mutable std::mutex mutex; + bool premultiplied = false; + explicit Impl(const AVFrame* source) : frame(AllocateFrame()) { + Validate(source); + Check(av_frame_ref(frame.get(), source), "Retain Vulkan frame"); + } +}; +VulkanFrameImage::VulkanFrameImage(const AVFrame* frame, bool premultiplied) : impl(new Impl(frame)) { impl->premultiplied = premultiplied; } +bool VulkanFrameImage::IsPremultiplied() const { return impl->premultiplied; } +VulkanFrameImage::~VulkanFrameImage() = default; +int VulkanFrameImage::GetWidth() const { return impl->frame->width; } +int VulkanFrameImage::GetHeight() const { return impl->frame->height; } +const AVFrame* VulkanFrameImage::GetFrame() const { return impl->frame.get(); } +int64_t VulkanFrameImage::GetBytes() const { + const auto* frame = reinterpret_cast(impl->frame->data[0]); + int64_t bytes = 0; + for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i) { + if (frame->mem[i] == VK_NULL_HANDLE) continue; + bool counted = false; + for (int j = 0; j < i; ++j) counted |= frame->mem[j] == frame->mem[i]; + if (counted) continue; + // Planes may share one allocation. Count its full padded allocation once. + size_t allocation = frame->size[i]; + for (int j = i + 1; j < AV_NUM_DATA_POINTERS; ++j) + if (frame->mem[j] == frame->mem[i]) allocation = std::max(allocation, frame->size[j]); + if (allocation > static_cast(std::numeric_limits::max() - bytes)) + return std::numeric_limits::max(); + bytes += static_cast(allocation); + } + return bytes; +} +QImage VulkanFrameImage::Materialize() const { + std::lock_guard lock(impl->mutex); + auto cpu = AllocateFrame(); + Check(av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0), "Read back Vulkan frame"); + Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); + QImage result(GetWidth(), GetHeight(), QImage::Format_RGBA8888); + if (result.isNull()) throw std::bad_alloc(); + auto* scaler = sws_getContext(cpu->width, cpu->height, AVPixelFormat(cpu->format), + result.width(), result.height(), AV_PIX_FMT_RGBA, SWS_BILINEAR, nullptr, nullptr, nullptr); + if (!scaler) throw std::runtime_error("Unable to create Vulkan readback color converter"); + std::unique_ptr owned_scaler(scaler, sws_freeContext); + int space = SWS_CS_DEFAULT; + switch (cpu->colorspace) { + case AVCOL_SPC_BT709: space = SWS_CS_ITU709; break; + case AVCOL_SPC_BT2020_NCL: case AVCOL_SPC_BT2020_CL: space = SWS_CS_BT2020; break; + case AVCOL_SPC_SMPTE240M: space = SWS_CS_SMPTE240M; break; + default: break; + } + const int* coefficients = sws_getCoefficients(space); + Check(sws_setColorspaceDetails(scaler, coefficients, cpu->color_range == AVCOL_RANGE_JPEG, + coefficients, 1, 0, 1 << 16, 1 << 16), "Set readback color conversion"); + uint8_t* output[] = {result.bits(), nullptr, nullptr, nullptr}; + int strides[] = {result.bytesPerLine(), 0, 0, 0}; + Check(sws_scale(scaler, cpu->data, cpu->linesize, 0, cpu->height, output, strides), "Convert readback pixels"); + if (impl->premultiplied) return QImage(result.constBits(), result.width(), result.height(), result.bytesPerLine(), QImage::Format_RGBA8888_Premultiplied).copy(); + return result.convertToFormat(QImage::Format_RGBA8888_Premultiplied); +} +struct VulkanFrameProcessor::Impl {}; +VulkanFrameProcessor::VulkanFrameProcessor() : impl(new Impl) {} +VulkanFrameProcessor::~VulkanFrameProcessor() = default; +void VulkanFrameProcessor::Reset() {} +std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* frame, int width, int height, bool high_quality) { + Validate(frame); + if (width <= 0 || height <= 0) throw std::invalid_argument("Invalid Vulkan output dimensions"); + // Preserve the existing FFmpegReader conversion matrix (SWS_CS_DEFAULT / BT.601). + // The generic compositor still honors metadata for callers outside that reader. + auto compatible = AllocateFrame(); + Check(av_frame_ref(compatible.get(), frame), "Reference reader Vulkan frame"); + compatible->colorspace = AVCOL_SPC_SMPTE170M; + auto source = std::make_shared(compatible.get()); + QTransform matrix; + matrix.scale(double(width) / frame->width, double(height) / frame->height); + return RenderVulkanImage(source, nullptr, width, height, matrix, 1.0, Qt::transparent, high_quality); +} +} diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h new file mode 100644 index 000000000..88194ad36 --- /dev/null +++ b/src/VulkanFrameImage.h @@ -0,0 +1,41 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_VULKAN_FRAME_IMAGE_H +#define OPENSHOT_VULKAN_FRAME_IMAGE_H +#include "FrameImage.h" +#include +struct AVFrame; +namespace openshot { +/// A retained FFmpeg Vulkan image. CPU pixels are created only by Materialize(). +class VulkanFrameImage final : public FrameImage { +public: + explicit VulkanFrameImage(const AVFrame* frame, bool premultiplied = false); + ~VulkanFrameImage() override; + VulkanFrameImage(const VulkanFrameImage&) = delete; + VulkanFrameImage& operator=(const VulkanFrameImage&) = delete; + int GetWidth() const override; + int GetHeight() const override; + int64_t GetBytes() const override; + QImage Materialize() const override; + /// Borrowed immutable frame; retain a reference before asynchronous use. + const AVFrame* GetFrame() const; + bool IsPremultiplied() const; +private: + struct Impl; + std::unique_ptr impl; +}; +/// Reuses the scaling pipeline across frames. Outputs own independent storage. +class VulkanFrameProcessor { +public: + VulkanFrameProcessor(); + ~VulkanFrameProcessor(); + VulkanFrameProcessor(const VulkanFrameProcessor&) = delete; + VulkanFrameProcessor& operator=(const VulkanFrameProcessor&) = delete; + std::shared_ptr Process(const AVFrame* frame, int width, int height, bool high_quality = false); + void Reset(); +private: + struct Impl; + std::unique_ptr impl; +}; +} +#endif diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp new file mode 100644 index 000000000..98e684cac --- /dev/null +++ b/src/vulkan/affine.comp @@ -0,0 +1,72 @@ +#version 450 +layout(local_size_x=16, local_size_y=16) in; +layout(binding=0) uniform sampler2D plane_y; +layout(binding=1) uniform sampler2D plane_u; +layout(binding=2) uniform sampler2D plane_v; +layout(binding=3) uniform sampler2D background_tex; +layout(binding=4, rgba8) uniform writeonly image2D output_img; +layout(push_constant) uniform Parameters { + vec4 matrix_x; + vec4 matrix_y; + vec4 color_coeffs; // Y multiplier, V->R, U->G, V->G + vec4 extra; // U->B, Y offset, chroma offset, opacity + vec4 clear; + ivec4 sizes; // output width/height, source width/height + ivec4 modes; // 0 planar, 1 NV12, 2 premult RGBA, 3 straight RGBA; background enabled +} p; +// Mitchell/Netravali family B=0, C=0.6 matches libswscale's default +// SWS_BICUBIC kernel. Expand the footprint on minification to suppress aliasing. +float cubic(float value) { + float x=abs(value); + if(x<1.0) return ((1.4*x-2.4)*x)*x+1.0; + if(x<2.0) return ((-0.6*x+3.0)*x-4.8)*x+2.4; + return 0.0; +} +vec4 sample_source(sampler2D image, vec2 uv) { + if(p.modes.z==0) return texture(image,uv); + ivec2 size=textureSize(image,0); + vec2 pos=uv*vec2(size)-0.5; + vec2 footprint=max(vec2(1.0),vec2(length(p.matrix_x.xy),length(p.matrix_y.xy)) + *vec2(size)/vec2(p.sizes.zw)); + ivec2 first=ivec2(ceil(pos-2.0*footprint)); + ivec2 last=ivec2(floor(pos+2.0*footprint)); + vec4 result=vec4(0); + float total=0.0; + for(int y=first.y;y<=last.y;++y) { + float wy=cubic((float(y)-pos.y)/footprint.y); + if(abs(wy)<0.000001) continue; + for(int x=first.x;x<=last.x;++x) { + float weight=wy*cubic((float(x)-pos.x)/footprint.x); + if(abs(weight)<0.000001) continue; + result+=texelFetch(image,clamp(ivec2(x,y),ivec2(0),size-1),0)*weight; + total+=weight; + } + } + return result/total; +} +void main() { + ivec2 pixel=ivec2(gl_GlobalInvocationID.xy); + if(any(greaterThanEqual(pixel,p.sizes.xy))) return; + vec2 center=vec2(pixel)+0.5; + vec2 src=vec2(dot(p.matrix_x.xyz,vec3(center,1)),dot(p.matrix_y.xyz,vec3(center,1))); + vec4 front=vec4(0); + if(all(greaterThanEqual(src,vec2(0))) && all(lessThan(src,vec2(p.sizes.zw)))) { + vec2 uv=src/vec2(p.sizes.zw); + if(p.modes.x>=2) { + front=sample_source(plane_y,uv); + front=clamp(front,0.0,1.0); + if(p.modes.x==3) front.rgb*=front.a; + else front.rgb=min(front.rgb,vec3(front.a)); + } else { + float y=(sample_source(plane_y,uv).r-p.extra.y)*p.color_coeffs.x; + vec2 chroma=p.modes.x==1 ? sample_source(plane_u,uv).rg : vec2(sample_source(plane_u,uv).r,sample_source(plane_v,uv).r); + chroma-=p.extra.z; + front=vec4(clamp(vec3(y+p.color_coeffs.y*chroma.y, + y+p.color_coeffs.z*chroma.x+p.color_coeffs.w*chroma.y, + y+p.extra.x*chroma.x),0,1),1); + } + front*=p.extra.w; + } + vec4 back=p.modes.y==1 ? texture(background_tex,center/vec2(p.sizes.xy)) : p.clear; + imageStore(output_img,pixel,front+back*(1-front.a)); +} diff --git a/src/vulkan/encode_nv12.comp b/src/vulkan/encode_nv12.comp new file mode 100644 index 000000000..02da26015 --- /dev/null +++ b/src/vulkan/encode_nv12.comp @@ -0,0 +1,28 @@ +#version 450 +layout(local_size_x=16,local_size_y=16) in; +layout(binding=0) uniform sampler2D source_image; +layout(binding=3,r8) uniform writeonly image2D output_y; +layout(binding=4,rg8) uniform writeonly image2D output_uv; +layout(push_constant) uniform Parameters { + vec4 matrix_x, matrix_y, coeffs, extra, clear; + ivec4 sizes, modes; +} p; +// Match the writer's existing SWS_CS_DEFAULT (BT.601), limited-range output. +vec3 yuv(vec3 rgb) { + return vec3(dot(rgb,vec3(0.299,0.587,0.114))*219.0/255.0+16.0/255.0, + dot(rgb,vec3(-0.168736,-0.331264,0.5))*224.0/255.0+128.0/255.0, + dot(rgb,vec3(0.5,-0.418688,-0.081312))*224.0/255.0+128.0/255.0); +} +void main() { + ivec2 at=ivec2(gl_GlobalInvocationID.xy); + if(any(greaterThanEqual(at,p.sizes.xy))) return; + vec3 value=yuv(texelFetch(source_image,at,0).rgb); + imageStore(output_y,at,vec4(value.x,0,0,1)); + if((at.x%2==0)&&(at.y%2==0)) { + vec3 rgb=vec3(0); + for(int y=0;y<2;++y) for(int x=0;x<2;++x) + rgb+=texelFetch(source_image,min(at+ivec2(x,y),p.sizes.xy-1),0).rgb; + vec3 chroma=yuv(rgb*0.25); + imageStore(output_uv,at/2,vec4(chroma.yz,0,1)); + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 48b5a9c37..296962f55 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -64,6 +64,7 @@ set(OPENSHOT_TESTS QtImageReader QtTextReader ReaderBase + RendererBase NativeArm64ProcessOracle Settings ScreenCaptureReader @@ -97,6 +98,10 @@ set(OPENSHOT_TESTS WaveEffect ) +if(ENABLE_GPU_FRAME AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND OPENSHOT_TESTS VulkanPresentation VulkanCompositor) +endif() + # The SIGPIPE crash-handler test is POSIX-only. Do not create an empty # Catch2 executable on Windows, where discovery registers a NOT_BUILT test. if(NOT WIN32) diff --git a/tests/CacheMemory.cpp b/tests/CacheMemory.cpp index 6338cac88..62774c406 100644 --- a/tests/CacheMemory.cpp +++ b/tests/CacheMemory.cpp @@ -21,6 +21,41 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class CacheTestSurface : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 16; } + int GetHeight() const override { return 16; } + int64_t GetBytes() const override { return 1024; } + QImage Materialize() const override { + ++reads; + QImage image(16, 16, QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::red); + return image; + } +}; +} +TEST_CASE("Hardware cache honors byte budget below the CPU frame floor", "[cache][gpu]") { + CacheMemory cache; + auto surface = std::make_shared(); + for (int i = 1; i <= 5; ++i) { + auto frame = std::make_shared(); + frame->number = i; + frame->AddDeferredImage(surface); + cache.SetMaxBytes(frame->GetBytes() * 2); + cache.Add(frame); + } + CHECK(cache.Count() == 2); + CHECK(cache.GetBytes() <= cache.GetMaxBytes()); + CHECK(surface->reads == 0); + CHECK_FALSE(cache.GetFrame(1)); + REQUIRE(cache.GetFrame(5)); + CHECK(cache.GetFrame(5)->GetImage()->pixelColor(0, 0) == QColor(Qt::red)); +} +#endif + TEST_CASE( "default constructor", "[libopenshot][cachememory]" ) { // Create cache object diff --git a/tests/Clip.cpp b/tests/Clip.cpp index 413cd2fbb..48e7e8403 100644 --- a/tests/Clip.cpp +++ b/tests/Clip.cpp @@ -46,6 +46,58 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class AudioOnlyGpuTestReader : public DummyReader { +public: + AudioOnlyGpuTestReader() : DummyReader(Fraction(24, 1), 16, 16, 48000, 2, 5.0) { + info.has_video = false; + info.has_audio = true; + } + std::shared_ptr GetFrame(int64_t n) override { + auto frame = std::make_shared(n, 16, 16, "#00000000", 2000, 2); + frame->SampleRate(48000); + frame->ChannelsLayout(LAYOUT_STEREO); + frame->AddAudioSilence(2000); + return frame; + } +}; +class ClipTestSurface : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 16; } + int GetHeight() const override { return 16; } + int64_t GetBytes() const override { return 1024; } + QImage Materialize() const override { + ++reads; + QImage image(16, 16, QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::red); + return image; + } +}; +} +TEST_CASE("Audio-only clip preserves deferred background", "[clip][gpu]") { + AudioOnlyGpuTestReader reader; + Clip clip(&reader); + clip.Open(); + auto surface = std::make_shared(); + auto background = std::make_shared(); + background->AddDeferredImage(surface); + auto result = clip.GetFrame(background, 1); + CHECK(surface->reads == 0); + CHECK(background->GetDeferredImage() == surface); + CHECK(result->GetDeferredImage() == surface); + reader.info.has_video = true; + clip.has_video = Keyframe(0.0); + TimelineInfoStruct options{}; + auto audio = clip.GetFrame(background, 2, &options); + CHECK(surface->reads == 0); + CHECK(background->GetDeferredImage() == surface); + CHECK(audio->GetAudioSamplesCount() == 2000); + clip.Close(); +} +#endif + namespace { struct ClipTransformCase { ScaleType scale; diff --git a/tests/Frame.cpp b/tests/Frame.cpp index e352293f6..c64e8a35b 100644 --- a/tests/Frame.cpp +++ b/tests/Frame.cpp @@ -277,3 +277,79 @@ TEST_CASE( "Convert_Image", "[libopenshot][opencv][frame]" ) CHECK(cvimage.channels() == 3); } #endif + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class TestFrameImage : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 8; } + int GetHeight() const override { return 6; } + int64_t GetBytes() const override { return 512; } + QImage Materialize() const override { + ++reads; + QImage result(8, 6, QImage::Format_RGBA8888_Premultiplied); + result.fill(Qt::red); + return result; + } +}; +} + +TEST_CASE("Deferred frame images preserve ownership and materialize lazily", "[Frame]") { + auto surface = std::make_shared(); + Frame frame; + const auto baseline_bytes = frame.GetBytes(); + frame.AddDeferredImage(surface); + CHECK(frame.GetWidth() == 8); + CHECK(frame.GetHeight() == 6); + CHECK(frame.has_image_data); + CHECK(frame.GetBytes() == baseline_bytes + 512); + Frame copy(frame); + CHECK(copy.GetDeferredImage() == surface); + CHECK(surface->reads == 0); + auto pixels = frame.GetImage(); + CHECK(surface->reads == 1); + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(frame.GetImage() == pixels); + CHECK(surface->reads == 1); + pixels->fill(Qt::blue); + CHECK(copy.GetImage()->pixelColor(0, 0) == QColor(Qt::red)); + CHECK(surface->reads == 2); +} + +TEST_CASE("Replacing deferred frame images does not read discarded pixels", "[Frame]") { + auto surface = std::make_shared(); + Frame frame; + frame.AddDeferredImage(surface); + frame.AddColor(Qt::green); + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(surface->reads == 0); + frame.AddDeferredImage(surface); + auto image = std::make_shared(4, 4, QImage::Format_RGBA8888_Premultiplied); + image->fill(Qt::blue); + frame.AddImage(image); + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(surface->reads == 0); + frame.AddDeferredImage(surface); + Frame empty; + frame = empty; + CHECK_FALSE(frame.GetDeferredImage()); + CHECK(surface->reads == 0); + CHECK(frame.GetImage()->pixelColor(0, 0) == QColor(Qt::black)); +} + +TEST_CASE("Deferred frame pixel access and field writes materialize correctly", "[Frame]") { + auto surface = std::make_shared(); + Frame frame; + frame.AddDeferredImage(surface); + CHECK(frame.GetPixels()[0] == 255); + CHECK(surface->reads == 1); + frame.AddDeferredImage(surface); + auto field = std::make_shared(8, 6, QImage::Format_RGBA8888_Premultiplied); + field->fill(Qt::blue); + frame.AddImage(field, true); + CHECK(surface->reads == 2); + CHECK(frame.GetImage()->pixelColor(0, 0) == QColor(Qt::red)); + CHECK(frame.GetImage()->pixelColor(0, 1) == QColor(Qt::blue)); +} +#endif diff --git a/tests/FrameMapper.cpp b/tests/FrameMapper.cpp index 781203efb..3d87ed614 100644 --- a/tests/FrameMapper.cpp +++ b/tests/FrameMapper.cpp @@ -27,6 +27,47 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +class MapperSurface : public FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 16; } + int GetHeight() const override { return 16; } + int64_t GetBytes() const override { return 1024; } + QImage Materialize() const override { + ++reads; + QImage result(16, 16, QImage::Format_RGBA8888_Premultiplied); + result.fill(Qt::red); + return result; + } +}; +class SurfaceReader : public DummyReader { +public: + std::shared_ptr surface = std::make_shared(); + SurfaceReader() : DummyReader(Fraction(24, 1), 16, 16, 48000, 2, 5.0) {} + std::shared_ptr GetFrame(int64_t number) override { + auto frame = DummyReader::GetFrame(number); + frame->AddDeferredImage(surface); + return frame; + } +}; +} + +TEST_CASE("Progressive frame mapping retains deferred image storage", "[libopenshot][framemapper]") { + SurfaceReader reader; + FrameMapper mapper(&reader, Fraction(30, 1), PULLDOWN_NONE, 48000, 2, LAYOUT_STEREO); + mapper.Open(); + for (int i = 1; i <= 8; ++i) { + auto frame = mapper.GetFrame(i); + REQUIRE(frame->GetDeferredImage() == reader.surface); + CHECK(frame->number == i); + } + CHECK(reader.surface->reads == 0); + mapper.Close(); +} +#endif + TEST_CASE( "NoOp_GetMappedFrame", "[libopenshot][framemapper]" ) { // Create a reader diff --git a/tests/RendererBase.cpp b/tests/RendererBase.cpp new file mode 100644 index 000000000..5aae0731a --- /dev/null +++ b/tests/RendererBase.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "RendererBase.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "Qt/VideoRenderer.h" +#include +#include +#endif + +namespace { +class ImageRenderer : public openshot::RendererBase { +public: + std::shared_ptr received; + void OverrideWidget(uintptr_t) override {} +protected: + void render(std::shared_ptr image) override { received = image; } +}; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +class RetainedRenderer : public ImageRenderer { +public: + std::shared_ptr frame; +protected: + void renderFrame(const std::shared_ptr& value) override { frame = value; } +}; +class DeferredImage : public openshot::FrameImage { +public: + mutable int reads = 0; + int GetWidth() const override { return 2; } + int GetHeight() const override { return 2; } + int64_t GetBytes() const override { return 16; } + QImage Materialize() const override { + ++reads; + QImage result(2, 2, QImage::Format_RGBA8888_Premultiplied); + result.fill(Qt::red); + return result; + } +}; +class NativePresenter : public QWidget, public FramePresenter { +public: + std::shared_ptr received; + void PresentFrame(const std::shared_ptr& frame) override { received = frame; } +}; +#endif +} + +TEST_CASE("Image renderer preserves CPU presentation", "[renderer]") { + ImageRenderer renderer; + renderer.paint(nullptr); + CHECK_FALSE(renderer.received); + auto frame = std::make_shared(); + frame->AddColor(2, 2, "#ff0000"); + renderer.paint(frame); + REQUIRE(renderer.received); + CHECK(renderer.received->pixelColor(0, 0) == QColor(Qt::red)); +} + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +TEST_CASE("Frame renderer retains surfaces until CPU fallback is requested", "[renderer]") { + auto surface = std::make_shared(); + auto frame = std::make_shared(); + frame->AddDeferredImage(surface); + RetainedRenderer retained; + retained.paint(frame); + CHECK(retained.frame == frame); + CHECK(surface->reads == 0); + CHECK_FALSE(retained.received); + ImageRenderer fallback; + fallback.paint(frame); + CHECK(surface->reads == 1); + REQUIRE(fallback.received); + CHECK(fallback.received->pixelColor(0, 0) == QColor(Qt::red)); +} + +TEST_CASE("Qt presenter queues owned frames without materializing pixels", "[renderer]") { + if (qEnvironmentVariableIsEmpty("DISPLAY") && qEnvironmentVariableIsEmpty("QT_QPA_PLATFORM")) + qputenv("QT_QPA_PLATFORM", "offscreen"); + int argc = 1; + char name[] = "renderer-test"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + NativePresenter widget; + VideoRenderer renderer; + renderer.OverrideWidget(reinterpret_cast(&widget)); + auto surface = std::make_shared(); + auto frame = std::make_shared(); + frame->AddDeferredImage(surface); + std::weak_ptr lifetime = frame; + renderer.paint(frame); + frame.reset(); + CHECK_FALSE(lifetime.expired()); + CHECK_FALSE(widget.received); + QCoreApplication::processEvents(); + REQUIRE(widget.received); + CHECK(surface->reads == 0); + CHECK(widget.received->GetDeferredImage() == surface); +} +#endif diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp new file mode 100644 index 000000000..8d12337f5 --- /dev/null +++ b/tests/VulkanCompositor.cpp @@ -0,0 +1,238 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "VulkanEncodeConverter.h" +#include "FFmpegReader.h" +#include "Clip.h" +#include "DummyReader.h" +#include +#include +#include +extern "C" { +#include +#include +} +namespace { +struct BufferDelete { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; +struct FrameDelete { void operator()(AVFrame* p) const { av_frame_free(&p); } }; +using Buffer=std::unique_ptr; +using Frame=std::unique_ptr; +class PatternReader : public openshot::DummyReader { + std::shared_ptr surface; + QImage pixels; +public: + PatternReader(std::shared_ptr image, bool gpu) + : DummyReader(openshot::Fraction(24,1),image->GetWidth(),image->GetHeight(),48000,2,5.0), + surface(gpu?image:nullptr),pixels(gpu?QImage():image->Materialize()) {} + std::shared_ptr GetFrame(int64_t n) override { + auto frame=std::make_shared(n,info.width,info.height,"#00000000"); + if(surface) frame->AddDeferredImage(surface); + else frame->AddImage(std::make_shared(pixels)); + return frame; + } +}; +} +TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan][hardware]") { + const char* enabled=std::getenv("OPENSHOT_TEST_VULKAN"); + if(!enabled || std::string(enabled)!="1") { + WARN("Hardware test not run: set OPENSHOT_TEST_VULKAN=1 and select a Vulkan device"); + return; + } + AVBufferRef* device_ptr=nullptr; + REQUIRE(av_hwdevice_ctx_create(&device_ptr,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)>=0); + Buffer device(device_ptr); + Buffer pool(av_hwframe_ctx_alloc(device.get())); + REQUIRE(pool); + auto* context=reinterpret_cast(pool->data); + context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; + context->width=16;context->height=16; + REQUIRE(av_hwframe_ctx_init(pool.get())>=0); + Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); + REQUIRE(cpu);REQUIRE(gpu); + cpu->format=AV_PIX_FMT_RGBA;cpu->width=16;cpu->height=16; + REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); + for(int y=0;y<16;++y)for(int x=0;x<16;++x){ + auto* p=cpu->data[0]+y*cpu->linesize[0]+x*4; + p[0]=255;p[1]=0;p[2]=0;p[3]=255; + } + REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); + REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); + auto source=std::make_shared(gpu.get()); + CHECK(source->GetBytes() >= 16 * 16 * 4); + QTransform transform;transform.translate(8,8); + auto first=openshot::RenderVulkanImage(source,nullptr,32,32,transform,0.5,Qt::blue); + // A second render must not overwrite the first cached frame's GPU storage. + auto second=openshot::RenderVulkanImage(source,nullptr,32,32,QTransform(),1.0,Qt::green); + gpu.reset();cpu.reset();pool.reset();device.reset();source.reset(); + CHECK(first->GetBytes() >= 32 * 32 * 4); + QImage image=first->Materialize(); + const auto blended=image.pixelColor(12,12); + CHECK(blended.red()>=126);CHECK(blended.red()<=129); + CHECK(blended.green()==0); + CHECK(blended.blue()>=126);CHECK(blended.blue()<=129); + CHECK(blended.alpha()==255); + CHECK(image.pixelColor(0,0)==QColor(Qt::blue)); + CHECK(second->Materialize().pixelColor(0,0)==QColor(Qt::red)); + // Export conversion remains on the GPU and produces limited-range BT.601 NV12. + auto encoded = openshot::ConvertVulkanToNV12(second); + auto encoded_surface = std::dynamic_pointer_cast(encoded); + REQUIRE(encoded_surface); + Frame downloaded(av_frame_alloc()); + REQUIRE(downloaded); + REQUIRE(av_hwframe_transfer_data(downloaded.get(), encoded_surface->GetFrame(), 0) >= 0); + REQUIRE(downloaded->format == AV_PIX_FMT_NV12); + const int y = downloaded->data[0][4 * downloaded->linesize[0] + 4]; + const int u = downloaded->data[1][2 * downloaded->linesize[1] + 4]; + const int v = downloaded->data[1][2 * downloaded->linesize[1] + 5]; + CHECK(y >= 80); CHECK(y <= 83); + CHECK(u >= 89); CHECK(u <= 92); + CHECK(v >= 238); CHECK(v <= 241); + const auto converted_red = encoded->Materialize().pixelColor(4, 4); + CHECK(converted_red.red() >= 252); + CHECK(converted_red.green() <= 3); + CHECK(converted_red.blue() <= 3); + // Exercise the actual Clip transform path against its QPainter fallback. + // Compare flat interior pixels; edge resampling intentionally differs. + for(auto mode : {openshot::SCALE_FIT,openshot::SCALE_CROP,openshot::SCALE_STRETCH,openshot::SCALE_NONE}) { + for(int gravity=openshot::GRAVITY_TOP_LEFT;gravity<=openshot::GRAVITY_BOTTOM_RIGHT;++gravity) { + PatternReader cpu_reader(second,false),gpu_reader(second,true); + openshot::Clip cpu_clip(&cpu_reader),gpu_clip(&gpu_reader); + for(auto* clip : {&cpu_clip,&gpu_clip}) { + clip->scale=mode; + clip->gravity=static_cast(gravity); + clip->scale_x=openshot::Keyframe(0.7);clip->scale_y=openshot::Keyframe(0.6); + clip->shear_x=openshot::Keyframe(0.12);clip->shear_y=openshot::Keyframe(-0.08); + clip->rotation=openshot::Keyframe(13.0);clip->alpha=openshot::Keyframe(0.6); + clip->Open(); + } + auto cpu_background=std::make_shared(1,80,60,"#00000000"); + auto gpu_background=std::make_shared(1,80,60,"#00000000"); + auto cpu_frame=cpu_clip.GetFrame(cpu_background,1); + auto gpu_frame=gpu_clip.GetFrame(gpu_background,1); + REQUIRE(gpu_frame->GetDeferredImage()); + auto expected=cpu_frame->GetImage();auto actual=gpu_frame->GetImage(); + REQUIRE(expected->size()==actual->size()); + int compared=0,max_error=0; + for(int y=2;yheight()-2;++y)for(int x=2;xwidth()-2;++x) { + const auto pixel=expected->pixel(x,y); + if(qAlpha(pixel)==0) continue; + bool interior=true; + for(int dy=-2;dy<=2;++dy)for(int dx=-2;dx<=2;++dx) + interior &= expected->pixel(x+dx,y+dy)==pixel; + if(!interior) continue; + const auto* a=actual->constScanLine(y)+4*x; + const auto* b=expected->constScanLine(y)+4*x; + for(int channel=0;channel<4;++channel)max_error=std::max(max_error,std::abs(int(a[channel])-int(b[channel]))); + ++compared; + } + INFO("scale=" << mode << " gravity=" << gravity << " compared=" << compared << " max_error=" << max_error); + CHECK(compared>8);CHECK(max_error<=3); + cpu_clip.Close();gpu_clip.Close(); + } + } + // Premultiplied output is not multiplied twice when materialized. + auto transparent=openshot::RenderVulkanImage(second,nullptr,32,32,QTransform(),0.5); + auto rgba=transparent->Materialize(); + const auto red=rgba.pixelColor(0,0); + CHECK(red.red()>=253); + CHECK(red.alpha()>=126);CHECK(red.alpha()<=129); +} + +TEST_CASE("Vulkan sequential decoding preserves pending audio video frames", "[vulkan][hardware][reader]") { + const char* enabled=std::getenv("OPENSHOT_TEST_VULKAN"); + const char* video=std::getenv("OPENSHOT_VULKAN_TEST_VIDEO"); + if(!enabled || std::string(enabled)!="1" || !video || !*video) { + WARN("Set OPENSHOT_TEST_VULKAN=1 and OPENSHOT_VULKAN_TEST_VIDEO to a 4K audio/video clip of at least 120 frames"); + return; + } + // Full-resolution RGBA frames exceed a small working-cache byte cap before + // their audio arrives. Evicting those pending frames silently produced black + // images and incorrect audio; compare sequential reads against direct seeks. + std::map> reference; + { + openshot::FFmpegReader reader(video); + reader.Open(); + for(int n : {12,120}) { + auto frame=reader.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + reference[n]=frame->GetImage(); + } + reader.Close(); + } + openshot::FFmpegReader reader(video); + reader.Open(); + for(int n=1;n<=120;++n) { + auto frame=reader.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + if(reference.count(n)) { + auto actual=frame->GetImage(); + const auto expected=reference[n]; + REQUIRE(actual->size()==expected->size()); + REQUIRE(actual->format()==expected->format()); + uint64_t error=0; + for(int y=0;yheight();++y) { + const auto* a=actual->constScanLine(y); + const auto* b=expected->constScanLine(y); + for(int x=0;xwidth()*4;++x) error+=std::abs(int(a[x])-int(b[x])); + } + const double mae=double(error)/(actual->width()*actual->height()*4.0); + INFO("frame=" << n << " sequential/direct MAE=" << mae); + CHECK(mae<=3.0); + } + } + reader.Close(); +} + +TEST_CASE("Vulkan high quality reader resizing uses bicubic antialiasing", "[vulkan][hardware][quality]") { + const char* enabled=std::getenv("OPENSHOT_TEST_VULKAN"); + if(!enabled || std::string(enabled)!="1") { + WARN("Set OPENSHOT_TEST_VULKAN=1 to compare GPU bicubic scaling against libswscale"); + return; + } + AVBufferRef* raw_device=nullptr; + REQUIRE(av_hwdevice_ctx_create(&raw_device,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)>=0); + Buffer device(raw_device),pool(av_hwframe_ctx_alloc(raw_device)); + REQUIRE(pool); + auto* context=reinterpret_cast(pool->data); + context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; + context->width=context->height=32; + REQUIRE(av_hwframe_ctx_init(pool.get())>=0); + Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); + REQUIRE(cpu);REQUIRE(gpu); + cpu->format=AV_PIX_FMT_RGBA;cpu->width=cpu->height=32; + REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); + for(int y=0;y<32;++y)for(int x=0;x<32;++x){ + auto* pixel=cpu->data[0]+y*cpu->linesize[0]+x*4; + pixel[0]=32+192*((x/4+y/4)%2);pixel[1]=y*8;pixel[2]=x*8;pixel[3]=255; + } + REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); + REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); + openshot::VulkanFrameProcessor processor; + for(const auto size : {QSize(12,10),QSize(61,47),QSize(32,32)}) { + QImage reference(size,QImage::Format_RGBA8888); + auto* scaler=sws_getContext(32,32,AV_PIX_FMT_RGBA,size.width(),size.height(),AV_PIX_FMT_RGBA, + SWS_BICUBIC,nullptr,nullptr,nullptr); + REQUIRE(scaler); + std::unique_ptr owner(scaler,sws_freeContext); + uint8_t* output[]={reference.bits(),nullptr,nullptr,nullptr}; + int strides[]={reference.bytesPerLine(),0,0,0}; + REQUIRE(sws_scale(scaler,cpu->data,cpu->linesize,0,32,output,strides)==size.height()); + auto high=processor.Process(gpu.get(),size.width(),size.height(),true); + REQUIRE(std::dynamic_pointer_cast(high)); + auto high_image=high->Materialize(); + auto low_image=processor.Process(gpu.get(),size.width(),size.height(),false)->Materialize(); + double high_error=0,low_error=0; + for(int y=0;y +#include +#include +#include +extern "C" { +#include +} +namespace { +class Preview : public QOpenGLWidget { +public: + openshot::VulkanOpenGLPresenter presenter{reinterpret_cast(this)}; + bool nativeDraw = false; + void paintGL() override { + QPainter painter(this); + painter.fillRect(rect(), Qt::blue); + painter.beginNativePainting(); + nativeDraw = presenter.Draw(0, 0, width() * devicePixelRatioF(), height() * devicePixelRatioF()); + painter.endNativePainting(); + painter.fillRect(QRect(0, 0, 20, 20), Qt::green); + } +}; +struct Frames { + AVBufferRef* device = nullptr; + AVBufferRef* pool = nullptr; + AVFrame* cpu = av_frame_alloc(); + AVFrame* gpu = av_frame_alloc(); + ~Frames() { + av_frame_free(&cpu); + av_frame_free(&gpu); + av_buffer_unref(&pool); + av_buffer_unref(&device); + } +}; +} +TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[vulkan][presentation]") { + if (!qEnvironmentVariableIsSet("OPENSHOT_TEST_VULKAN_PRESENTATION")) { + WARN("Set OPENSHOT_TEST_VULKAN_PRESENTATION=1 to run GPU presentation oracle"); + return; + } + int argc = 1; + char name[] = "vulkan-presentation"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + Frames data; + REQUIRE(av_hwdevice_ctx_create(&data.device, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) >= 0); + data.pool = av_hwframe_ctx_alloc(data.device); + REQUIRE(data.pool); + auto pool = reinterpret_cast(data.pool->data); + pool->format = AV_PIX_FMT_VULKAN; + pool->sw_format = AV_PIX_FMT_RGBA; + pool->width = pool->height = 64; + REQUIRE(av_hwframe_ctx_init(data.pool) >= 0); + REQUIRE(av_hwframe_get_buffer(data.pool, data.gpu, 0) >= 0); + data.cpu->format = AV_PIX_FMT_RGBA; + data.cpu->width = data.cpu->height = 64; + REQUIRE(av_frame_get_buffer(data.cpu, 32) >= 0); + for (int y = 0; y < 64; ++y) { + for (int x = 0; x < 64; ++x) { + auto p = data.cpu->data[0] + y * data.cpu->linesize[0] + x * 4; + p[0] = y < 32 ? 255 : 0; p[1] = 0; p[2] = y < 32 ? 0 : 255; p[3] = 255; + } + } + REQUIRE(av_hwframe_transfer_data(data.gpu, data.cpu, 0) >= 0); + auto frame = std::make_shared(); + frame->AddDeferredImage(std::make_shared(data.gpu, true)); + Preview widget; + widget.resize(128, 128); + VideoRenderer renderer; + renderer.OverrideWidget(reinterpret_cast(&widget)); + widget.show(); + renderer.paint(frame); + for (int i = 0; i < 20; ++i) { + QCoreApplication::processEvents(); + QThread::msleep(5); + } + QImage image; + // Exercise both semaphore directions repeatedly; a single frame can hide + // a race where Vulkan consumes its own release signal before GL waits. + for (int i = 0; i < 64; ++i) { + image = widget.grabFramebuffer(); + REQUIRE(widget.nativeDraw); + REQUIRE(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); + REQUIRE(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); + } + REQUIRE(widget.nativeDraw); + REQUIRE(frame->GetDeferredImage()); + REQUIRE_FALSE(image.isNull()); + if (qEnvironmentVariableIsSet("OPENSHOT_PRESENTATION_CAPTURE")) + image.save(qEnvironmentVariable("OPENSHOT_PRESENTATION_CAPTURE")); + INFO("center=" << image.pixelColor(image.width()/2, image.height()/2).name().toStdString() + << " overlay=" << image.pixelColor(5, 5).name().toStdString()); + CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); + CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); + CHECK(image.pixelColor(5, 5) == QColor(Qt::green)); +} From 0f79e668f785ab86450ca7d66ec1503a65e90578 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 27 Oct 2026 23:57:47 -0500 Subject: [PATCH 02/50] Embed Vulkan shaders in libopenshot Compile the compute shaders into C headers at build time instead of loading SPIR-V from absolute build-directory paths, so installed and packaged GPU builds work. --- src/CMakeLists.txt | 28 +++++++++++++--------------- src/VulkanCompositor.cpp | 14 ++++++-------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 258465259..a991b6e60 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -457,22 +457,20 @@ if(ENABLE_GPU_FRAME) if(NOT GLSLANG_VALIDATOR) message(FATAL_ERROR "ENABLE_GPU_FRAME requires glslangValidator") endif() - set(VULKAN_AFFINE_SHADER "${CMAKE_CURRENT_BINARY_DIR}/affine.comp.spv") - add_custom_command(OUTPUT "${VULKAN_AFFINE_SHADER}" - COMMAND "${GLSLANG_VALIDATOR}" -V -S comp -o "${VULKAN_AFFINE_SHADER}" - "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/affine.comp" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/affine.comp" VERBATIM) - add_custom_target(openshot-vulkan-shaders DEPENDS "${VULKAN_AFFINE_SHADER}") - set(VULKAN_ENCODE_SHADER "${CMAKE_CURRENT_BINARY_DIR}/encode_nv12.comp.spv") - add_custom_command(OUTPUT "${VULKAN_ENCODE_SHADER}" - COMMAND "${GLSLANG_VALIDATOR}" -V -S comp -o "${VULKAN_ENCODE_SHADER}" - "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/encode_nv12.comp" - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/encode_nv12.comp" VERBATIM) - add_custom_target(openshot-vulkan-encode-shaders DEPENDS "${VULKAN_ENCODE_SHADER}") - add_dependencies(openshot openshot-vulkan-encode-shaders) - target_compile_definitions(openshot PRIVATE OPENSHOT_VULKAN_ENCODE_SHADER_PATH="${VULKAN_ENCODE_SHADER}") + # Embed SPIR-V in the library so installed builds need no shader files. + set(VULKAN_SHADER_HEADERS) + file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/vulkan") + foreach(shader affine encode_nv12) + set(shader_header "${CMAKE_CURRENT_BINARY_DIR}/vulkan/${shader}.comp.h") + add_custom_command(OUTPUT "${shader_header}" + COMMAND "${GLSLANG_VALIDATOR}" -V -S comp --vn "openshot_${shader}_spv" + -o "${shader_header}" "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/${shader}.comp" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/${shader}.comp" VERBATIM) + list(APPEND VULKAN_SHADER_HEADERS "${shader_header}") + endforeach() + add_custom_target(openshot-vulkan-shaders DEPENDS ${VULKAN_SHADER_HEADERS}) add_dependencies(openshot openshot-vulkan-shaders) - target_compile_definitions(openshot PRIVATE OPENSHOT_VULKAN_AFFINE_SHADER_PATH="${VULKAN_AFFINE_SHADER}") + target_include_directories(openshot PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/vulkan") target_link_libraries(openshot PRIVATE Vulkan::Vulkan) find_package(PkgConfig QUIET) if(PKG_CONFIG_FOUND) diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 88b09057a..a4edda06b 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -5,7 +5,7 @@ #include "VulkanFrameImage.h" #include #include -#include +#include #include #include #include @@ -14,6 +14,9 @@ extern "C" { #include #include } +// SPIR-V generated from src/vulkan/*.comp at build time. +#include "affine.comp.h" +#include "encode_nv12.comp.h" namespace openshot { namespace { void VkCheck(VkResult r, const char* operation) { @@ -101,14 +104,9 @@ class Pipeline { si.magFilter=si.minFilter=VK_FILTER_LINEAR; si.addressModeU=si.addressModeV=si.addressModeW=VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; VkCheck(vkCreateSampler(device,&si,nullptr,&sampler),"Create sampler"); - std::ifstream shader(encode_nv12?OPENSHOT_VULKAN_ENCODE_SHADER_PATH:OPENSHOT_VULKAN_AFFINE_SHADER_PATH,std::ios::binary|std::ios::ate); - if(!shader) throw std::runtime_error("Vulkan affine shader unavailable"); - const auto bytes=shader.tellg(); - if(bytes<=0 || bytes%4) throw std::runtime_error("Invalid Vulkan affine shader"); - std::vector code(static_cast(bytes)/4); - shader.seekg(0); shader.read(reinterpret_cast(code.data()),bytes); VkShaderModuleCreateInfo mi{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; - mi.codeSize=code.size()*4; mi.pCode=code.data(); + mi.codeSize=encode_nv12?sizeof(openshot_encode_nv12_spv):sizeof(openshot_affine_spv); + mi.pCode=encode_nv12?openshot_encode_nv12_spv:openshot_affine_spv; VkShaderModule module; VkCheck(vkCreateShaderModule(device,&mi,nullptr,&module),"Create shader module"); VkComputePipelineCreateInfo pci{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; From 142505e08c6a5fbfb022af93487b5a9a5815bb90 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 27 Oct 2026 23:57:47 -0500 Subject: [PATCH 03/50] Fall back to the CPU pipeline when Vulkan is off or fails - OPENSHOT_RENDER_BACKEND=cpu now disables Vulkan decoding at runtime in a GPU build, so every frame follows the unchanged CPU path. Output is identical to a CPU build. - A failed GPU export no longer aborts. The writer reads the frame back and encodes the rest of the export from CPU pixels. - Compare clip composite modes against openshot::CompositeType. --- src/Clip.cpp | 4 ++-- src/FFmpegReader.cpp | 8 +++++--- src/FFmpegWriter.cpp | 17 +++++++++++------ src/FFmpegWriter.h | 3 +++ src/VulkanFrameImage.cpp | 6 ++++++ src/VulkanFrameImage.h | 2 ++ tests/FFmpegReader.cpp | 29 +++++++++++++++++++++++++++++ 7 files changed, 58 insertions(+), 11 deletions(-) diff --git a/src/Clip.cpp b/src/Clip.cpp index 7a06f32ab..b66642268 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -1375,7 +1375,7 @@ void Clip::apply_background(std::shared_ptr frame, } return; } - if (composite == QPainter::CompositionMode_SourceOver) { + if (composite == COMPOSITE_SOURCE_OVER) { auto source = frame->GetDeferredImage(); auto background = background_frame->GetDeferredImage(); QColor clear; @@ -1444,7 +1444,7 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { } #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (composite == QPainter::CompositionMode_SourceOver && + if (composite == COMPOSITE_SOURCE_OVER && corner_radius.GetValue(frame->number) == 0.0 && display == FRAME_DISPLAY_NONE) { if (auto source = std::dynamic_pointer_cast(frame->GetDeferredImage())) { try { diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 5ab2c9bb8..2ba6cc5f3 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -279,7 +279,9 @@ void FFmpegReader::Open() { { hw_de_on = (!force_sw_decode && openshot::Settings::Instance()->HARDWARE_DECODER != 0 ? 1 : 0); #ifdef OPENSHOT_ENABLE_GPU_FRAME - hw_de_on = !force_sw_decode; + // GPU builds decode through Vulkan unless the CPU backend is selected + // at runtime. Decode failures reopen this reader in software. + hw_de_on = !force_sw_decode && VulkanRenderingEnabled(); vulkan_processor.reset(); #endif hw_decode_failed = false; @@ -469,7 +471,7 @@ void FFmpegReader::Open() { #endif hw_device_ctx = NULL; // Here the first hardware initialisations are made - #ifdef OPENSHOT_ENABLE_GPU_FRAME +#ifdef OPENSHOT_ENABLE_GPU_FRAME if (CreateVulkanDecodeDevice(&hw_device_ctx) >= 0) { #else if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { @@ -1730,7 +1732,7 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL if (hw_de_on && hw_de_supported) { int err; - #ifdef OPENSHOT_ENABLE_GPU_FRAME +#ifdef OPENSHOT_ENABLE_GPU_FRAME if (next_frame2->format == AV_PIX_FMT_VULKAN) { decoded_frame = next_frame2; } else diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 229143792..6d8413854 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -106,6 +106,9 @@ void FFmpegWriter::Open() { if (!is_open) { // Open the writer is_open = true; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + gpu_export_failed = false; +#endif // Prepare streams (if needed) if (!prepare_streams) @@ -2621,7 +2624,7 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { #if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL // NVENC accepts encoder-owned CUDA surfaces. Preserve GPU residency through // RGB->NV12 conversion and the Vulkan/CUDA device-to-device transfer. - if (hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx && + if (!gpu_export_failed && hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx && video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA) { auto surface = frame->GetDeferredImage(); if (surface) { @@ -2644,11 +2647,13 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { add_avframe(frame, encoded); Logger::Instance()->AppendDebugMethod("Vulkan export GPU transfer to NVENC", "frame", frame->number); return; - } catch (...) { + } catch (const std::exception& error) { av_frame_free(&encoded); - // Failed interop may leave external semaphore ownership uncertain. - // Surface the error instead of attempting CPU readback on that surface. - throw; + // Interop only touches the temporary scaled/NV12 surfaces, and every + // compute pass reading the frame's own surface has completed. Read + // back that surface below and keep this export on the CPU path. + gpu_export_failed = true; + Logger::Instance()->Log(std::string("Vulkan export CPU fallback: ") + error.what()); } } } @@ -2872,7 +2877,7 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra #if USE_HW_ACCEL if (hw_en_on && hw_en_supported) { - #ifdef OPENSHOT_ENABLE_GPU_FRAME +#ifdef OPENSHOT_ENABLE_GPU_FRAME ret = avcodec_send_frame(video_codec_ctx, frame_final->hw_frames_ctx ? frame_final : hw_frame); #else ret = avcodec_send_frame(video_codec_ctx, hw_frame); diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 1859cb866..bef75e7fe 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -140,6 +140,9 @@ namespace openshot { AVFrame *persistent_dst_frame = nullptr; uint8_t *persistent_dst_buffer = nullptr; int persistent_dst_size = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + bool gpu_export_failed = false; ///< Encode remaining frames from CPU pixels +#endif int audio_outbuf_size; int audio_input_frame_size; diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 4d3b71fd5..6ed7ecf35 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -4,6 +4,8 @@ #include "VulkanCompositor.h" #include #include +#include +#include #include #include #include @@ -38,6 +40,10 @@ void Validate(const AVFrame* frame) { throw std::invalid_argument("Expected an FFmpeg Vulkan hardware frame"); } } +bool VulkanRenderingEnabled() { + const char* backend = std::getenv("OPENSHOT_RENDER_BACKEND"); + return !(backend && std::strcmp(backend, "cpu") == 0); +} struct VulkanFrameImage::Impl { OwnedFrame frame; mutable std::mutex mutex; diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h index 88194ad36..b133fd42c 100644 --- a/src/VulkanFrameImage.h +++ b/src/VulkanFrameImage.h @@ -6,6 +6,8 @@ #include struct AVFrame; namespace openshot { +/// False when OPENSHOT_RENDER_BACKEND=cpu selects the CPU pipeline at runtime. +bool VulkanRenderingEnabled(); /// A retained FFmpeg Vulkan image. CPU pixels are created only by Materialize(). class VulkanFrameImage final : public FrameImage { public: diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index a91e73a16..a27a5810e 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -95,6 +95,28 @@ struct HardwareDecoderSettingsGuard { } }; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +struct RenderBackendGuard { + std::string previous; + bool had_previous = false; + + explicit RenderBackendGuard(const char* backend) { + if (const char* value = std::getenv("OPENSHOT_RENDER_BACKEND")) { + previous = value; + had_previous = true; + } + setenv("OPENSHOT_RENDER_BACKEND", backend, 1); + } + + ~RenderBackendGuard() { + if (had_previous) + setenv("OPENSHOT_RENDER_BACKEND", previous.c_str(), 1); + else + unsetenv("OPENSHOT_RENDER_BACKEND"); + } +}; +#endif + struct TemporaryFileGuard { std::string path; @@ -916,6 +938,10 @@ TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[li HardwareDecoderSettingsGuard guard; Settings::Instance()->HARDWARE_DECODER = 0; Settings::Instance()->HW_DE_DEVICE_SET = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // GPU builds select Vulkan decoding by render backend, not by decoder setting. + RenderBackendGuard backend("cpu"); +#endif std::stringstream path; path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; @@ -926,6 +952,9 @@ TEST_CASE( "HardwareDecodeSuccessful_IsFalse_WhenHardwareDecodeIsDisabled", "[li auto frame = r.GetFrame(1); REQUIRE(frame->has_image_data); CHECK_FALSE(r.HardwareDecodeSuccessful()); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + CHECK_FALSE(frame->GetDeferredImage()); +#endif r.Close(); } From 8804add4e927b26c6b35827ba3718ae2148b40a5 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 27 Oct 2026 23:57:47 -0500 Subject: [PATCH 04/50] Run Vulkan tests automatically and report skips Vulkan tests no longer need opt-in environment variables. They detect a Vulkan device, display and OpenGL interop themselves and print a "Skipping Vulkan test:" message that CTest reports as skipped, not passed. The reader cache test uses bundled media by default, and the renderer test waits for queued delivery on any Qt platform. --- tests/CMakeLists.txt | 8 +++++++- tests/RendererBase.cpp | 6 +++++- tests/VulkanCompositor.cpp | 37 ++++++++++++++++++++---------------- tests/VulkanPresentation.cpp | 26 +++++++++++++++++++++---- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 296962f55..3abed91ed 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -185,7 +185,13 @@ foreach(tname ${OPENSHOT_TESTS}) SKIP_REGULAR_EXPRESSION "Skipping VAAPI test:" ) endif() - if(tname STREQUAL "QtTextReader") + if(tname MATCHES "^Vulkan") + # Report missing GPU, display or interop support as skipped, not passed. + list(APPEND test_properties + SKIP_REGULAR_EXPRESSION "Skipping Vulkan test:" + ) + endif() + if(tname STREQUAL "QtTextReader" OR tname STREQUAL "RendererBase") # Pixel assertions need a font engine; Qt's minimal plugin has no font database. list(APPEND test_properties ENVIRONMENT "QT_QPA_PLATFORM=offscreen") elseif(tname STREQUAL "Caption" OR tname STREQUAL "Timer") diff --git a/tests/RendererBase.cpp b/tests/RendererBase.cpp index 5aae0731a..b353335e0 100644 --- a/tests/RendererBase.cpp +++ b/tests/RendererBase.cpp @@ -5,6 +5,7 @@ #ifdef OPENSHOT_ENABLE_GPU_FRAME #include "Qt/VideoRenderer.h" #include +#include #include #endif @@ -90,7 +91,10 @@ TEST_CASE("Qt presenter queues owned frames without materializing pixels", "[ren frame.reset(); CHECK_FALSE(lifetime.expired()); CHECK_FALSE(widget.received); - QCoreApplication::processEvents(); + // Platform plugins may need several passes to deliver the queued frame. + QDeadlineTimer deadline(1000); + while (!widget.received && !deadline.hasExpired()) + QCoreApplication::processEvents(QEventLoop::AllEvents, 10); REQUIRE(widget.received); CHECK(surface->reads == 0); CHECK(widget.received->GetDeferredImage() == surface); diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 8d12337f5..034c8a5c2 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -19,6 +19,20 @@ struct BufferDelete { void operator()(AVBufferRef* p) const { av_buffer_unref(&p struct FrameDelete { void operator()(AVFrame* p) const { av_frame_free(&p); } }; using Buffer=std::unique_ptr; using Frame=std::unique_ptr; +// CTest reports this message as skipped; see tests/CMakeLists.txt. +bool VulkanAvailable() { + if(!openshot::VulkanRenderingEnabled()) { + WARN("Skipping Vulkan test: OPENSHOT_RENDER_BACKEND=cpu"); + return false; + } + AVBufferRef* device=nullptr; + if(av_hwdevice_ctx_create(&device,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)<0) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return false; + } + av_buffer_unref(&device); + return true; +} class PatternReader : public openshot::DummyReader { std::shared_ptr surface; QImage pixels; @@ -35,11 +49,7 @@ class PatternReader : public openshot::DummyReader { }; } TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan][hardware]") { - const char* enabled=std::getenv("OPENSHOT_TEST_VULKAN"); - if(!enabled || std::string(enabled)!="1") { - WARN("Hardware test not run: set OPENSHOT_TEST_VULKAN=1 and select a Vulkan device"); - return; - } + if(!VulkanAvailable()) return; AVBufferRef* device_ptr=nullptr; REQUIRE(av_hwdevice_ctx_create(&device_ptr,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)>=0); Buffer device(device_ptr); @@ -141,12 +151,11 @@ TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan] } TEST_CASE("Vulkan sequential decoding preserves pending audio video frames", "[vulkan][hardware][reader]") { - const char* enabled=std::getenv("OPENSHOT_TEST_VULKAN"); - const char* video=std::getenv("OPENSHOT_VULKAN_TEST_VIDEO"); - if(!enabled || std::string(enabled)!="1" || !video || !*video) { - WARN("Set OPENSHOT_TEST_VULKAN=1 and OPENSHOT_VULKAN_TEST_VIDEO to a 4K audio/video clip of at least 120 frames"); - return; - } + if(!VulkanAvailable()) return; + // OPENSHOT_VULKAN_TEST_VIDEO may select a larger (e.g. 4K) audio/video clip + // of at least 120 frames; the bundled trailer covers the same cache paths. + std::string video=TEST_MEDIA_PATH "sintel_trailer-720p.mp4"; + if(const char* custom=std::getenv("OPENSHOT_VULKAN_TEST_VIDEO"); custom && *custom) video=custom; // Full-resolution RGBA frames exceed a small working-cache byte cap before // their audio arrives. Evicting those pending frames silently produced black // images and incorrect audio; compare sequential reads against direct seeks. @@ -186,11 +195,7 @@ TEST_CASE("Vulkan sequential decoding preserves pending audio video frames", "[v } TEST_CASE("Vulkan high quality reader resizing uses bicubic antialiasing", "[vulkan][hardware][quality]") { - const char* enabled=std::getenv("OPENSHOT_TEST_VULKAN"); - if(!enabled || std::string(enabled)!="1") { - WARN("Set OPENSHOT_TEST_VULKAN=1 to compare GPU bicubic scaling against libswscale"); - return; - } + if(!VulkanAvailable()) return; AVBufferRef* raw_device=nullptr; REQUIRE(av_hwdevice_ctx_create(&raw_device,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)>=0); Buffer device(raw_device),pool(av_hwframe_ctx_alloc(raw_device)); diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 28d1185da..b066d829d 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -6,6 +6,7 @@ #include "Qt/VideoRenderer.h" #include "Qt/VulkanOpenGLPresenter.h" #include +#include #include #include #include @@ -40,16 +41,26 @@ struct Frames { }; } TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[vulkan][presentation]") { - if (!qEnvironmentVariableIsSet("OPENSHOT_TEST_VULKAN_PRESENTATION")) { - WARN("Set OPENSHOT_TEST_VULKAN_PRESENTATION=1 to run GPU presentation oracle"); + // CTest reports "Skipping Vulkan test:" as skipped; see tests/CMakeLists.txt. + if (!openshot::VulkanRenderingEnabled()) { + WARN("Skipping Vulkan test: OPENSHOT_RENDER_BACKEND=cpu"); + return; + } + const QByteArray platform = qgetenv("QT_QPA_PLATFORM"); + if ((platform.isEmpty() || platform.startsWith("xcb")) && !qEnvironmentVariableIsSet("DISPLAY") && + !qEnvironmentVariableIsSet("WAYLAND_DISPLAY")) { + WARN("Skipping Vulkan test: presentation needs a display with OpenGL"); + return; + } + Frames data; + if (av_hwdevice_ctx_create(&data.device, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) < 0) { + WARN("Skipping Vulkan test: no Vulkan device is available"); return; } int argc = 1; char name[] = "vulkan-presentation"; char* argv[] = {name, nullptr}; QApplication app(argc, argv); - Frames data; - REQUIRE(av_hwdevice_ctx_create(&data.device, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) >= 0); data.pool = av_hwframe_ctx_alloc(data.device); REQUIRE(data.pool); auto pool = reinterpret_cast(data.pool->data); @@ -75,6 +86,13 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ VideoRenderer renderer; renderer.OverrideWidget(reinterpret_cast(&widget)); widget.show(); + QCoreApplication::processEvents(); + auto* context = widget.context(); + if (!context || !context->hasExtension("GL_EXT_memory_object_fd") || + !context->hasExtension("GL_EXT_semaphore_fd")) { + WARN("Skipping Vulkan test: OpenGL lacks Vulkan memory/semaphore interop"); + return; + } renderer.paint(frame); for (int i = 0; i < 20; ++i) { QCoreApplication::processEvents(); From 80ccd6ef8d8e07b3efb07a7dc83e3cb585a1e9fc Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 28 Oct 2026 14:03:12 -0500 Subject: [PATCH 05/50] Keep Vulkan compositor passes in flight instead of waiting on each Each pipeline now reuses a small ring of command buffers, fences and descriptor sets. A submit no longer blocks the CPU; later readers wait on the frames' timeline semaphores, and each slot keeps its frames alive until the GPU is done with them. --- src/FFmpegWriter.cpp | 6 +-- src/VulkanCompositor.cpp | 87 +++++++++++++++++++++++++++------------- 2 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 6d8413854..2a52e8186 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -2649,9 +2649,9 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { return; } catch (const std::exception& error) { av_frame_free(&encoded); - // Interop only touches the temporary scaled/NV12 surfaces, and every - // compute pass reading the frame's own surface has completed. Read - // back that surface below and keep this export on the CPU path. + // Interop only touches the temporary scaled/NV12 surfaces. The readback + // below waits on the frame surface's semaphores, so it sees finished + // pixels; keep this export on the CPU path. gpu_export_failed = true; Logger::Instance()->Log(std::string("Vulkan export CPU fallback: ") + error.what()); } diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index a4edda06b..4465568c3 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -27,6 +27,8 @@ void AvCheck(int r, const char* operation) { } struct FrameDelete { void operator()(AVFrame* f) const { av_frame_free(&f); } }; using FramePtr=std::unique_ptr; +// Submissions in flight per pipeline before Render waits for the oldest one. +constexpr int kSlots=4; struct Parameters { float matrix_x[4], matrix_y[4], coeffs[4], extra[4], clear[4]; int sizes[4], modes[4]; @@ -56,7 +58,18 @@ class Pipeline { VkPipelineLayout layout=VK_NULL_HANDLE; VkPipeline pipeline=VK_NULL_HANDLE; VkSampler sampler=VK_NULL_HANDLE; - VkDescriptorSet set=VK_NULL_HANDLE; + // One in-flight submission. Consumers order themselves through the frames' + // timeline semaphores; the slot only keeps resources alive until it retires. + struct Slot { + VkDescriptorSet set=VK_NULL_HANDLE; + VkCommandBuffer cmd=VK_NULL_HANDLE; + VkFence fence=VK_NULL_HANDLE; + bool busy=false; + std::vector views; + std::vector frames; + }; + std::array ring; + int next_slot=0; int width,height; bool encode_nv12; public: @@ -93,13 +106,20 @@ class Pipeline { VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; li.setLayoutCount=1; li.pSetLayouts=&descriptors; li.pushConstantRangeCount=1; li.pPushConstantRanges=⦥ VkCheck(vkCreatePipelineLayout(device,&li,nullptr,&layout),"Create pipeline layout"); - std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,4},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2}}}; + std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,4*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2*kSlots}}}; VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - pi.maxSets=1; pi.poolSizeCount=2; pi.pPoolSizes=sizes.data(); + pi.maxSets=kSlots; pi.poolSizeCount=2; pi.pPoolSizes=sizes.data(); VkCheck(vkCreateDescriptorPool(device,&pi,nullptr,&pool),"Create descriptor pool"); - VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; - ai.descriptorPool=pool; ai.descriptorSetCount=1; ai.pSetLayouts=&descriptors; - VkCheck(vkAllocateDescriptorSets(device,&ai,&set),"Allocate descriptors"); + for(auto& slot:ring){ + VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + ai.descriptorPool=pool; ai.descriptorSetCount=1; ai.pSetLayouts=&descriptors; + VkCheck(vkAllocateDescriptorSets(device,&ai,&slot.set),"Allocate descriptors"); + VkCommandBufferAllocateInfo ci{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ci.commandPool=commands; ci.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY; ci.commandBufferCount=1; + VkCheck(vkAllocateCommandBuffers(device,&ci,&slot.cmd),"Allocate affine commands"); + VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkCheck(vkCreateFence(device,&fi,nullptr,&slot.fence),"Create affine fence"); + } VkSamplerCreateInfo si{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; si.magFilter=si.minFilter=VK_FILTER_LINEAR; si.addressModeU=si.addressModeV=si.addressModeW=VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; @@ -119,6 +139,10 @@ class Pipeline { } ~Pipeline(){Destroy();} void Destroy(){ + for(auto& slot:ring){ + Retire(slot); + if(slot.fence) vkDestroyFence(device,slot.fence,nullptr); + } if(pipeline) vkDestroyPipeline(device,pipeline,nullptr); if(sampler) vkDestroySampler(device,sampler,nullptr); if(pool) vkDestroyDescriptorPool(device,pool,nullptr); @@ -127,6 +151,17 @@ class Pipeline { if(commands) vkDestroyCommandPool(device,commands,nullptr); av_buffer_unref(&frames); } + // Wait for a slot's previous submission, then release what it kept alive. + void Retire(Slot& slot){ + if(slot.busy){ + vkWaitForFences(device,1,&slot.fence,VK_TRUE,UINT64_MAX); + vkResetFences(device,1,&slot.fence); + slot.busy=false; + } + for(auto view:slot.views) vkDestroyImageView(device,view,nullptr); + slot.views.clear(); + slot.frames.clear(); + } VkImageView View(VkImage image,VkFormat format,VkImageAspectFlags aspect){ VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; vi.image=image; vi.viewType=VK_IMAGE_VIEW_TYPE_2D; vi.format=format; @@ -188,10 +223,12 @@ class Pipeline { const bool separate_nv12 = p.modes[0]==1 && sf->img[1]!=VK_NULL_HANDLE && sf->img[1]!=sf->img[0]; if(separate_nv12) plane_count=2; const auto* svc=reinterpret_cast(src_fc->hwctx); - std::vector owned_views; - VkCommandBuffer cmd=VK_NULL_HANDLE; - VkFence fence=VK_NULL_HANDLE; - bool submitted=false; + Slot& slot=ring[next_slot]; + next_slot=(next_slot+1)%kSlots; + Retire(slot); + auto& owned_views=slot.views; + VkCommandBuffer cmd=slot.cmd; + const VkDescriptorSet set=slot.set; try { std::array views{}; auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ @@ -219,9 +256,6 @@ class Pipeline { writes[i].descriptorType=(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;writes[i].pImageInfo=&images[i]; } vkUpdateDescriptorSets(device,5,writes.data(),0,nullptr); - VkCommandBufferAllocateInfo ai{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - ai.commandPool=commands;ai.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY;ai.commandBufferCount=1; - VkCheck(vkAllocateCommandBuffers(device,&ai,&cmd),"Allocate affine commands"); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};bi.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; VkCheck(vkBeginCommandBuffer(cmd,&bi),"Begin affine commands"); std::vector semaphores; @@ -252,7 +286,6 @@ class Pipeline { vkCmdPushConstants(cmd,layout,VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(p),&p); vkCmdDispatch(cmd,(width+15)/16,(height+15)/16,1); VkCheck(vkEndCommandBuffer(cmd),"End affine commands"); - VkFenceCreateInfo fi{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};VkCheck(vkCreateFence(device,&fi,nullptr,&fence),"Create affine fence"); VkTimelineSemaphoreSubmitInfo ti{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; ti.waitSemaphoreValueCount=waits.size();ti.pWaitSemaphoreValues=waits.data();ti.signalSemaphoreValueCount=signal_values.size();ti.pSignalSemaphoreValues=signal_values.data(); std::vector stages(waits.size(),VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); @@ -260,9 +293,9 @@ class Pipeline { si.waitSemaphoreCount=semaphores.size();si.pWaitSemaphores=semaphores.data();si.pWaitDstStageMask=stages.data(); si.signalSemaphoreCount=semaphores.size();si.pSignalSemaphores=semaphores.data();si.commandBufferCount=1;si.pCommandBuffers=&cmd; vkcontext->lock_queue(device_context,queue_family,0); - auto result=vkQueueSubmit(queue,1,&si,fence); + auto result=vkQueueSubmit(queue,1,&si,slot.fence); vkcontext->unlock_queue(device_context,queue_family,0); - VkCheck(result,"Submit affine renderer");submitted=true; + VkCheck(result,"Submit affine renderer");slot.busy=true; for(size_t changed_index=0;changed_indexlayout[item.second]=VK_IMAGE_LAYOUT_GENERAL; @@ -270,20 +303,18 @@ class Pipeline { item.first->queue_family[item.second]=barriers[changed_index].dstQueueFamilyIndex; item.first->sem_value[item.second]++; } - // Publish semaphore values before releasing frame locks. Never hold a queue lock while waiting. + // Publish semaphore values before releasing frame locks. Do not wait here: + // later readers wait on those semaphores, and the slot keeps the frames alive. locks.clear(); - VkCheck(vkWaitForFences(device,1,&fence,VK_TRUE,UINT64_MAX),"Wait affine renderer"); - submitted=false; - vkDestroyFence(device,fence,nullptr);fence=VK_NULL_HANDLE; - vkFreeCommandBuffers(device,commands,1,&cmd);cmd=VK_NULL_HANDLE; - for(auto view:owned_views) vkDestroyImageView(device,view,nullptr); - owned_views.clear(); // A later allocation failure must not destroy views twice. - return std::make_shared(output.get(),!encode_nv12); + auto result_image=std::make_shared(output.get(),!encode_nv12); + for(auto* f:all){ + FramePtr ref(av_frame_clone(f)); + if(!ref) throw std::bad_alloc(); + slot.frames.push_back(std::move(ref)); + } + return result_image; }catch(...){ - if(submitted && fence) vkWaitForFences(device,1,&fence,VK_TRUE,UINT64_MAX); - if(fence) vkDestroyFence(device,fence,nullptr); - if(cmd) vkFreeCommandBuffers(device,commands,1,&cmd); - for(auto view:owned_views) vkDestroyImageView(device,view,nullptr); + Retire(slot); throw; } } From 094700ad52b511ca4087d32249e5b2d8c8c2ed22 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 28 Oct 2026 14:03:12 -0500 Subject: [PATCH 06/50] Transform and composite timeline layers in one Vulkan pass Timeline layers only need the composited result, so draw the transformed clip straight onto the background. Layers with after-keyframe effects or timeline effects keep the separate passes. --- src/Clip.cpp | 36 ++++++++++++++++++++++++++++++++++++ src/Clip.h | 6 ++++++ 2 files changed, 42 insertions(+) diff --git a/src/Clip.cpp b/src/Clip.cpp index b66642268..68b12b557 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -522,6 +522,12 @@ std::shared_ptr Clip::GetFrame(std::shared_ptr backgroun // Apply effects BEFORE applying keyframes (if any local or global effects are used) apply_effects(frame, timeline_frame_number, options, true); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Timeline layers only need the composited result, so skip the separate transform pass + if (options && background_frame && apply_vulkan_layer(frame, background_frame, timeline_size)) + return frame; +#endif + // Apply keyframe / transforms to current clip image apply_keyframes(frame, timeline_size); @@ -1435,6 +1441,36 @@ bool Clip::isNear(double a, double b) return fabs(a - b) < 0.000001; } +#ifdef OPENSHOT_ENABLE_GPU_FRAME +bool Clip::apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size) +{ + if (!frame->has_image_data || composite != COMPOSITE_SOURCE_OVER || + corner_radius.GetValue(frame->number) != 0.0 || display != FRAME_DISPLAY_NONE) + return false; + // Effects applied after keyframes (and timeline transitions/masks) need the transformed layer + for (auto effect : effects) + if (!effect->info.apply_before_clip) + return false; + if (timeline && !static_cast(timeline)->Effects().empty()) + return false; + + auto source = std::dynamic_pointer_cast(frame->GetDeferredImage()); + auto background = background_frame->GetDeferredImage(); + QColor clear; + if (!source || !(std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear))) + return false; + try { + background_frame->AddDeferredImage(RenderVulkanImage(source, background, timeline_size.width(), + timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()), + alpha.GetValue(frame->number), clear)); + return true; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan layer CPU fallback: ") + error.what()); + return false; + } +} +#endif + // Apply keyframes to the source frame (if any) void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { // Skip out if video was disabled or only an audio frame (no visualisation in use) diff --git a/src/Clip.h b/src/Clip.h index 072cb0c4e..c1150c3a7 100644 --- a/src/Clip.h +++ b/src/Clip.h @@ -145,6 +145,12 @@ namespace openshot { /// Apply keyframes to an openshot::Frame and use an existing background frame (if any) void apply_keyframes(std::shared_ptr frame, QSize timeline_size); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + /// Transform and composite onto the background in one Vulkan pass. Returns false + /// when an effect needs the transformed layer by itself, or the pass is unavailable. + bool apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size); +#endif + /// Apply waveform image to an openshot::Frame and use an existing background frame (if any) void apply_waveform(std::shared_ptr frame, QSize timeline_size); From 1fc7e020b2fb13166e96ecafc27444bbd034e223 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 28 Oct 2026 14:31:16 -0500 Subject: [PATCH 07/50] Share Vulkan output pools and allow releasing their memory Freed GPU frames return to FFmpeg pools that never shrink. Keep the compositor pools in one shared list, so ReleaseVulkanMemory() can drop them; FFmpeg frees each pool once its last frame is released. Submit slots no longer hold frame references, since FFmpeg waits on a frame's semaphores before freeing it. --- bindings/python/openshot.i | 2 + src/VulkanCompositor.cpp | 77 +++++++++++++++++++++++++------------- src/VulkanCompositor.h | 3 ++ 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index e6af31f13..389be7a07 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -112,6 +112,7 @@ class QWidget; #include "RendererBase.h" #if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) #include "Qt/VulkanOpenGLPresenter.h" +#include "VulkanCompositor.h" #endif #include "Settings.h" #include "TimelineBase.h" @@ -525,6 +526,7 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "RendererBase.h" #if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) %include "Qt/VulkanOpenGLPresenter.h" +namespace openshot { void ReleaseVulkanMemory(); } #endif %include "Settings.h" %include "TimelineBase.h" diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 4465568c3..4443bd7fa 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -29,6 +29,37 @@ struct FrameDelete { void operator()(AVFrame* f) const { av_frame_free(&f); } }; using FramePtr=std::unique_ptr; // Submissions in flight per pipeline before Render waits for the oldest one. constexpr int kSlots=4; +// Output pools are shared by every thread so ReleaseVulkanMemory() can drop them. +// A pool keeps freed frames for reuse; FFmpeg frees a dropped pool's memory once +// its last frame is released. +struct OutputPools { + std::mutex mutex; + std::map,AVBufferRef*> pools; + ~OutputPools(){ for(auto& item:pools) av_buffer_unref(&item.second); } +}; +OutputPools& SharedPools(){ static OutputPools pools; return pools; } +// Returns a new reference the caller must release. +AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ + auto& shared=SharedPools(); + const std::lock_guard lock(shared.mutex); + auto& pool=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12}]; + if(!pool){ + AVBufferRef* frames=av_hwframe_ctx_alloc(device_ref); + if(!frames) throw std::bad_alloc(); + auto* fc=reinterpret_cast(frames->data); + fc->format=AV_PIX_FMT_VULKAN; fc->sw_format=encode_nv12?AV_PIX_FMT_NV12:AV_PIX_FMT_RGBA; + fc->width=w; fc->height=h; + auto* vc=reinterpret_cast(fc->hwctx); + if(encode_nv12) vc->flags=AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + vc->usage=static_cast(VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT); + const int status=av_hwframe_ctx_init(frames); + if(status<0){ av_buffer_unref(&frames); AvCheck(status,"Initialize Vulkan compositor output"); } + pool=frames; + } + AVBufferRef* ref=av_buffer_ref(pool); + if(!ref) throw std::bad_alloc(); + return ref; +} struct Parameters { float matrix_x[4], matrix_y[4], coeffs[4], extra[4], clear[4]; int sizes[4], modes[4]; @@ -46,7 +77,7 @@ struct LockedFrame { ~LockedFrame(){vkcontext->unlock_frame(context,frame);} }; class Pipeline { - AVBufferRef* frames=nullptr; + AVBufferRef* device_ref=nullptr; AVHWDeviceContext* device_context=nullptr; AVVulkanDeviceContext* vkcontext=nullptr; VkDevice device=VK_NULL_HANDLE; @@ -59,36 +90,29 @@ class Pipeline { VkPipeline pipeline=VK_NULL_HANDLE; VkSampler sampler=VK_NULL_HANDLE; // One in-flight submission. Consumers order themselves through the frames' - // timeline semaphores; the slot only keeps resources alive until it retires. + // timeline semaphores, and FFmpeg waits on them before freeing a frame, so + // the slot only keeps its own commands and views until it retires. struct Slot { VkDescriptorSet set=VK_NULL_HANDLE; VkCommandBuffer cmd=VK_NULL_HANDLE; VkFence fence=VK_NULL_HANDLE; bool busy=false; std::vector views; - std::vector frames; }; std::array ring; int next_slot=0; int width,height; bool encode_nv12; public: - Pipeline(AVBufferRef* device_ref,int w,int h,bool encode=false):width(w),height(h),encode_nv12(encode) { + Pipeline(AVBufferRef* device_owner,int w,int h,bool encode=false):width(w),height(h),encode_nv12(encode) { + device_ref=av_buffer_ref(device_owner); + if(!device_ref) throw std::bad_alloc(); device_context=reinterpret_cast(device_ref->data); vkcontext=reinterpret_cast(device_context->hwctx); device=vkcontext->act_dev; queue_family=vkcontext->queue_family_comp_index; if(queue_family<0) throw std::runtime_error("No Vulkan compute queue"); try { - frames=av_hwframe_ctx_alloc(device_ref); - if(!frames) throw std::bad_alloc(); - auto* fc=reinterpret_cast(frames->data); - fc->format=AV_PIX_FMT_VULKAN; fc->sw_format=encode_nv12?AV_PIX_FMT_NV12:AV_PIX_FMT_RGBA; - fc->width=w; fc->height=h; - auto* vc=reinterpret_cast(fc->hwctx); - if(encode_nv12) vc->flags=AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; - vc->usage=static_cast(VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT); - AvCheck(av_hwframe_ctx_init(frames),"Initialize Vulkan compositor output"); vkGetDeviceQueue(device,queue_family,0,&queue); VkCommandPoolCreateInfo ci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; ci.queueFamilyIndex=queue_family; ci.flags=VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; @@ -149,9 +173,9 @@ class Pipeline { if(layout) vkDestroyPipelineLayout(device,layout,nullptr); if(descriptors) vkDestroyDescriptorSetLayout(device,descriptors,nullptr); if(commands) vkDestroyCommandPool(device,commands,nullptr); - av_buffer_unref(&frames); + av_buffer_unref(&device_ref); } - // Wait for a slot's previous submission, then release what it kept alive. + // Wait for a slot's previous submission, then release its views. void Retire(Slot& slot){ if(slot.busy){ vkWaitForFences(device,1,&slot.fence,VK_TRUE,UINT64_MAX); @@ -160,7 +184,6 @@ class Pipeline { } for(auto view:slot.views) vkDestroyImageView(device,view,nullptr); slot.views.clear(); - slot.frames.clear(); } VkImageView View(VkImage image,VkFormat format,VkImageAspectFlags aspect){ VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; @@ -183,7 +206,10 @@ class Pipeline { } FramePtr output(av_frame_alloc()); if(!output) throw std::bad_alloc(); - AvCheck(av_hwframe_get_buffer(frames,output.get(),0),"Allocate compositor output"); + AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12); + const int allocated=av_hwframe_get_buffer(pool,output.get(),0); + av_buffer_unref(&pool); + AvCheck(allocated,"Allocate compositor output"); AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); output->colorspace=encode_nv12?AVCOL_SPC_SMPTE170M:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; if(encode_nv12 && src_fc->sw_format!=AV_PIX_FMT_RGBA) throw std::runtime_error("Encoder conversion requires RGBA"); @@ -304,15 +330,9 @@ class Pipeline { item.first->sem_value[item.second]++; } // Publish semaphore values before releasing frame locks. Do not wait here: - // later readers wait on those semaphores, and the slot keeps the frames alive. + // later readers wait on those semaphores. locks.clear(); - auto result_image=std::make_shared(output.get(),!encode_nv12); - for(auto* f:all){ - FramePtr ref(av_frame_clone(f)); - if(!ref) throw std::bad_alloc(); - slot.frames.push_back(std::move(ref)); - } - return result_image; + return std::make_shared(output.get(),!encode_nv12); }catch(...){ Retire(slot); throw; @@ -363,4 +383,11 @@ std::shared_ptr ConvertVulkanToNV12(std::shared_ptrRender(*src,nullptr,QTransform(),1.0,Qt::black); } +void ReleaseVulkanMemory() { + auto& shared=SharedPools(); + const std::lock_guard lock(shared.mutex); + for(auto& item:shared.pools) av_buffer_unref(&item.second); + shared.pools.clear(); +} + } diff --git a/src/VulkanCompositor.h b/src/VulkanCompositor.h index b4988c63c..b5bbab487 100644 --- a/src/VulkanCompositor.h +++ b/src/VulkanCompositor.h @@ -14,5 +14,8 @@ std::shared_ptr RenderVulkanImage( std::shared_ptr source, std::shared_ptr background, int width, int height, const QTransform& source_to_output, double opacity, const QColor& clear = Qt::transparent, bool high_quality = false); +/// Return pooled GPU frame memory once its frames are released, i.e. after +/// clearing caches. Later renders create new pools as needed. +void ReleaseVulkanMemory(); } #endif From 4be2077645f6dd803a4b625e36a60c7e253123ed Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 28 Oct 2026 14:51:26 -0500 Subject: [PATCH 08/50] Wait less for audio before finishing reader frames A decoded frame was held until the audio stream was a full second past it, so every reader kept 1-2 seconds of decoded images. With GPU decoding that cost about 0.9 GB of video memory per 4K clip. A 0.25 second margin gives identical playback audio on 22 test files (AAC, MP3, Vorbis, Opus, FLAC, AC3, ALAC and PCM) and cuts that to about 0.34 GB. --- src/FFmpegReader.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 2ba6cc5f3..ac435276b 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -3004,8 +3004,10 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { } } + // Every frame waiting for its audio keeps a decoded image, so keep this margin + // small: 0.25 sec covers partial and delayed audio packets. double audio_pts_diff = audio_pts_seconds - frame_pts_seconds; - if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 1.0) + if ((frame_pts_seconds < audio_pts_seconds && audio_pts_diff > 0.25) || (recent_pts_diff > 1.5) || packet_status.audio_eof || packet_status.end_of_file) { // Audio stream is past this frame (so it must be done) From 1baecd22f99ae73d2111fb44d1f38aff74ea1101 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 28 Oct 2026 15:43:32 -0500 Subject: [PATCH 09/50] Start each GPU export with fresh frame pools FFmpeg binds a Vulkan frame to the first CUDA context it is sent to, and pooled frames were reused by the next export. The second export in a session then fell back to the CPU, and the third could crash. Release the pools when a writer opens and closes, and report export and decode fallbacks as warnings so they show in the console. --- src/FFmpegReader.cpp | 2 +- src/FFmpegWriter.cpp | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index ac435276b..327d29b60 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -2151,7 +2151,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { video_pts_seconds = (double(video_pts) * info.video_timebase.ToDouble()) + pts_offset_seconds; return; } catch (const std::exception& error) { - Logger::Instance()->Log(std::string("Vulkan processing failed; retrying software decode: ") + error.what()); + Logger::Instance()->Log(std::string("Vulkan processing failed; retrying software decode: ") + error.what(), Logger::LevelWarning); hw_decode_failed = true; RemoveAVFrame(pFrame); pFrame = nullptr; diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 2a52e8186..2f0b6be8a 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -108,6 +108,9 @@ void FFmpegWriter::Open() { is_open = true; #ifdef OPENSHOT_ENABLE_GPU_FRAME gpu_export_failed = false; + // FFmpeg binds a GPU frame to the first CUDA context it is sent to. Start + // each export with fresh pools, so no frame is tied to an old encoder. + ReleaseVulkanMemory(); #endif // Prepare streams (if needed) @@ -1477,6 +1480,10 @@ void FFmpegWriter::Close() { close_video(oc, video_st); if (audio_st) close_audio(oc, audio_st); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Free pooled frames still tied to this encoder's CUDA context + ReleaseVulkanMemory(); +#endif // Remove single software scaler if (img_convert_ctx) @@ -2653,7 +2660,7 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { // below waits on the frame surface's semaphores, so it sees finished // pixels; keep this export on the CPU path. gpu_export_failed = true; - Logger::Instance()->Log(std::string("Vulkan export CPU fallback: ") + error.what()); + Logger::Instance()->Log(std::string("Vulkan export CPU fallback: ") + error.what(), Logger::LevelWarning); } } } From 0dd3d1d9ea73685d6d2e34dee0df44d89c7993c1 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Oct 2026 00:39:04 -0500 Subject: [PATCH 10/50] Skip FFmpeg 7's zeroed spherical angles in the metadata tests FFmpeg 7's MP4 reader returns 0,0,0 orientation, like 8 and 9. The written angles are still checked directly in the file. --- tests/SphericalMetadata.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/SphericalMetadata.cpp b/tests/SphericalMetadata.cpp index 630a5475c..9227f8907 100644 --- a/tests/SphericalMetadata.cpp +++ b/tests/SphericalMetadata.cpp @@ -93,9 +93,9 @@ static void check_written_orientation(const std::string& path, static bool has_ffmpeg_orientation_bug(float yaw, float pitch, float roll) { const auto major = avformat_version() >> 16; - // libavformat 62 and 63 correspond to FFmpeg 8 and 9. - if ((major == 62 || major == 63) && yaw == 0 && pitch == 0 && roll == 0) { - WARN("Skipping spherical reader angle assertions: FFmpeg 8/9 returned 0,0,0. " + // libavformat 61, 62 and 63 correspond to FFmpeg 7, 8 and 9. + if (major >= 61 && major <= 63 && yaw == 0 && pitch == 0 && roll == 0) { + WARN("Skipping spherical reader angle assertions: FFmpeg 7-9 returned 0,0,0. " "Written MP4 angles were verified independently. Upstream patch submitted " "to ffmpeg-devel: avformat/mov: preserve spherical orientation when reading vexu. " "FFmpeg version: " << av_version_info()); From c71b9e07f7166e12d5a81012c535aaa04b1c0d4c Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Oct 2026 00:39:04 -0500 Subject: [PATCH 11/50] Build the Linux CI library with Vulkan and FFmpeg 7 The runner image now has FFmpeg 7.1.3 with Vulkan and newer Vulkan headers in /opt. The job fails if the system FFmpeg 4.2 is also linked. --- .gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3391b613e..d620419a1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,9 @@ linux-builder: variables: OPENCV_ROOT: /opt/opencv-4.13.0 OpenCV_DIR: /opt/opencv-4.13.0/lib/cmake/opencv4 + # Runner image provides FFmpeg 7 (with Vulkan) and newer Vulkan headers in /opt + FFMPEG_ROOT: /opt/ffmpeg-7.1.3 + VULKAN_HEADERS_ROOT: /opt/vulkan-headers artifacts: expire_in: 6 months paths: @@ -41,10 +44,16 @@ linux-builder: - export PKG_CONFIG_PATH="$OPENCV_ROOT/lib/pkgconfig:$PKG_CONFIG_PATH" - export LD_LIBRARY_PATH="$OPENCV_ROOT/lib:$LD_LIBRARY_PATH" - export PATH="$OPENCV_ROOT/bin:$PATH" + - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT (FFmpeg 7 with Vulkan)"; exit 1; fi + - . "$FFMPEG_ROOT/env.sh" + - export FFMPEGDIR="$FFMPEG_ROOT" - mkdir -p build; cd build; - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support" - - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON ../ + - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ - cmake --build . --parallel $(nproc) + # Some focal libraries link the system FFmpeg 4.2; never load two FFmpegs + - ldd src/libopenshot.so | grep -E 'libav|libsw|vulkan' + - if ldd src/libopenshot.so | grep -qE 'libavcodec\.so\.58|libavutil\.so\.56'; then echo "ERROR libopenshot also loads the system FFmpeg 4.2"; exit 1; fi - make install - ctest --output-on-failure -VV - make doc From e9029244cf3997b11df0723cb86177466bf33843 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Oct 2026 10:50:26 -0500 Subject: [PATCH 12/50] Pick a real GPU for Vulkan instead of FFmpeg's first device FFmpeg uses the first Vulkan device, which is often a software driver such as llvmpipe, so decoding quietly fell back to the CPU. Try GPUs by name, discrete first, skip software devices, and log the chosen device. OPENSHOT_VULKAN_DEVICE overrides the choice. --- src/FFmpegReader.cpp | 71 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 327d29b60..a717e4c15 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -30,6 +30,11 @@ #include "FFmpegReader.h" #ifdef OPENSHOT_ENABLE_GPU_FRAME #include "VulkanFrameImage.h" +extern "C" { +#include +} +#include +#include #endif #include "Exceptions.h" #include "MemoryTrim.h" @@ -143,6 +148,49 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 #if USE_HW_ACCEL #ifdef OPENSHOT_ENABLE_GPU_FRAME +// FFmpeg uses the first Vulkan device, which can be a software driver such as +// lavapipe. List real GPUs instead, discrete before integrated. Software (CPU) +// devices are skipped: they cannot decode video and are slower than our CPU path. +// OPENSHOT_VULKAN_DEVICE (an index or a name) overrides the choice. +static std::vector VulkanDeviceCandidates() +{ + const char* chosen = std::getenv("OPENSHOT_VULKAN_DEVICE"); + if (chosen && chosen[0]) + return {chosen}; + + std::vector> ranked; + VkApplicationInfo app = {}; + app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo create = {}; + create.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + create.pApplicationInfo = &app; + VkInstance instance = VK_NULL_HANDLE; + if (vkCreateInstance(&create, nullptr, &instance) != VK_SUCCESS) + return {""}; // Let FFmpeg report why Vulkan is unavailable + uint32_t count = 0; + vkEnumeratePhysicalDevices(instance, &count, nullptr); + std::vector devices(count); + vkEnumeratePhysicalDevices(instance, &count, devices.data()); + for (uint32_t i = 0; i < count; ++i) { + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(devices[i], &props); + if (props.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) + continue; + const int rank = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 0 + : props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 1 : 2; + // Select by name: FFmpeg's device indices can be in a different order + ranked.emplace_back(rank, props.deviceName); + } + vkDestroyInstance(instance, nullptr); + std::stable_sort(ranked.begin(), ranked.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + std::vector candidates; + for (const auto& device : ranked) + candidates.push_back(device.second); + return candidates; +} + // All readers share one device so their surfaces can be composed without interop. // Keep an owning reference for the process lifetime; each reader receives its own ref. static int CreateVulkanDecodeDevice(AVBufferRef** output) @@ -150,13 +198,32 @@ static int CreateVulkanDecodeDevice(AVBufferRef** output) struct SharedDevice { std::mutex mutex; AVBufferRef* device = nullptr; + bool tried = false; ~SharedDevice() { av_buffer_unref(&device); } }; static SharedDevice shared; const std::lock_guard lock(shared.mutex); if (!shared.device) { - int result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0); - if (result < 0) return result; + // Only search once; without a usable GPU every reader decodes on the CPU. + if (shared.tried) + return AVERROR(ENODEV); + shared.tried = true; + int result = AVERROR(ENODEV); + for (const auto& device : VulkanDeviceCandidates()) { + result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, + device.empty() ? nullptr : device.c_str(), nullptr, 0); + if (result >= 0) + break; + } + if (result < 0) { + Logger::Instance()->Log("No usable Vulkan GPU; decoding on the CPU", Logger::LevelWarning); + return result; + } + auto* hwctx = static_cast( + reinterpret_cast(shared.device->data)->hwctx); + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(hwctx->phys_dev, &props); + Logger::Instance()->Log(std::string("Vulkan device: ") + props.deviceName, Logger::LevelInfo); } *output = av_buffer_ref(shared.device); return *output ? 0 : AVERROR(ENOMEM); From 011aee9eb7bd7de41a6b72d2535a39986dd4714a Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Oct 2026 11:47:38 -0500 Subject: [PATCH 13/50] Share GPU frames with VA-API encoders and add GPU inspection helpers AMD and Intel exports no longer copy each frame through system memory. The final NV12 images use linear DMA-BUF memory that VA-API imports directly, and the VA-API encoder opens on the card that renders. If sharing fails, the export continues with the old copy path. New helpers for the UI: ListGpuDevices(), ActiveGpuDevice() and GpuEncoders(). The Vulkan tests now use the same device choice as the app, so they no longer skip when a software driver is listed first. --- bindings/python/openshot.i | 4 + src/CMakeLists.txt | 1 + src/FFmpegReader.cpp | 91 +----------- src/FFmpegWriter.cpp | 48 ++++-- src/FFmpegWriter.h | 1 + src/GpuDevices.cpp | 276 +++++++++++++++++++++++++++++++++++ src/GpuDevices.h | 37 +++++ src/VulkanCompositor.cpp | 110 ++++++++++++-- src/VulkanEncodeConverter.h | 3 +- tests/CMakeLists.txt | 2 + tests/VulkanCompositor.cpp | 84 ++++++++++- tests/VulkanPresentation.cpp | 3 +- 12 files changed, 546 insertions(+), 114 deletions(-) create mode 100644 src/GpuDevices.cpp create mode 100644 src/GpuDevices.h diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index 389be7a07..c65e5a04b 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -113,6 +113,7 @@ class QWidget; #if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) #include "Qt/VulkanOpenGLPresenter.h" #include "VulkanCompositor.h" +#include "GpuDevices.h" #endif #include "Settings.h" #include "TimelineBase.h" @@ -527,6 +528,9 @@ static int openshot_swig_is_qwidget(PyObject *obj) { #if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) %include "Qt/VulkanOpenGLPresenter.h" namespace openshot { void ReleaseVulkanMemory(); } +%include "GpuDevices.h" +%template(GpuDeviceVector) std::vector; +%template(StringVector) std::vector; #endif %include "Settings.h" %include "TimelineBase.h" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a991b6e60..eec506ee9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -449,6 +449,7 @@ if(ENABLE_GPU_FRAME) list(APPEND all_comps avfilter) target_sources(openshot PRIVATE VulkanFrameImage.cpp) target_sources(openshot PRIVATE VulkanCompositor.cpp) + target_sources(openshot PRIVATE GpuDevices.cpp) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_sources(openshot PRIVATE Qt/VulkanOpenGLPresenter.cpp Qt/VulkanOpenGLPresenter.h) endif() diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index a717e4c15..1e99e11fd 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -29,12 +29,8 @@ #include "FFmpegReader.h" #ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "GpuDevices.h" #include "VulkanFrameImage.h" -extern "C" { -#include -} -#include -#include #endif #include "Exceptions.h" #include "MemoryTrim.h" @@ -147,89 +143,6 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 #if USE_HW_ACCEL -#ifdef OPENSHOT_ENABLE_GPU_FRAME -// FFmpeg uses the first Vulkan device, which can be a software driver such as -// lavapipe. List real GPUs instead, discrete before integrated. Software (CPU) -// devices are skipped: they cannot decode video and are slower than our CPU path. -// OPENSHOT_VULKAN_DEVICE (an index or a name) overrides the choice. -static std::vector VulkanDeviceCandidates() -{ - const char* chosen = std::getenv("OPENSHOT_VULKAN_DEVICE"); - if (chosen && chosen[0]) - return {chosen}; - - std::vector> ranked; - VkApplicationInfo app = {}; - app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; - app.apiVersion = VK_API_VERSION_1_1; - VkInstanceCreateInfo create = {}; - create.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; - create.pApplicationInfo = &app; - VkInstance instance = VK_NULL_HANDLE; - if (vkCreateInstance(&create, nullptr, &instance) != VK_SUCCESS) - return {""}; // Let FFmpeg report why Vulkan is unavailable - uint32_t count = 0; - vkEnumeratePhysicalDevices(instance, &count, nullptr); - std::vector devices(count); - vkEnumeratePhysicalDevices(instance, &count, devices.data()); - for (uint32_t i = 0; i < count; ++i) { - VkPhysicalDeviceProperties props; - vkGetPhysicalDeviceProperties(devices[i], &props); - if (props.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) - continue; - const int rank = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 0 - : props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 1 : 2; - // Select by name: FFmpeg's device indices can be in a different order - ranked.emplace_back(rank, props.deviceName); - } - vkDestroyInstance(instance, nullptr); - std::stable_sort(ranked.begin(), ranked.end(), - [](const auto& a, const auto& b) { return a.first < b.first; }); - std::vector candidates; - for (const auto& device : ranked) - candidates.push_back(device.second); - return candidates; -} - -// All readers share one device so their surfaces can be composed without interop. -// Keep an owning reference for the process lifetime; each reader receives its own ref. -static int CreateVulkanDecodeDevice(AVBufferRef** output) -{ - struct SharedDevice { - std::mutex mutex; - AVBufferRef* device = nullptr; - bool tried = false; - ~SharedDevice() { av_buffer_unref(&device); } - }; - static SharedDevice shared; - const std::lock_guard lock(shared.mutex); - if (!shared.device) { - // Only search once; without a usable GPU every reader decodes on the CPU. - if (shared.tried) - return AVERROR(ENODEV); - shared.tried = true; - int result = AVERROR(ENODEV); - for (const auto& device : VulkanDeviceCandidates()) { - result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, - device.empty() ? nullptr : device.c_str(), nullptr, 0); - if (result >= 0) - break; - } - if (result < 0) { - Logger::Instance()->Log("No usable Vulkan GPU; decoding on the CPU", Logger::LevelWarning); - return result; - } - auto* hwctx = static_cast( - reinterpret_cast(shared.device->data)->hwctx); - VkPhysicalDeviceProperties props; - vkGetPhysicalDeviceProperties(hwctx->phys_dev, &props); - Logger::Instance()->Log(std::string("Vulkan device: ") + props.deviceName, Logger::LevelInfo); - } - *output = av_buffer_ref(shared.device); - return *output ? 0 : AVERROR(ENOMEM); -} -#endif - // Get hardware pix format static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) { @@ -539,7 +452,7 @@ void FFmpegReader::Open() { hw_device_ctx = NULL; // Here the first hardware initialisations are made #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (CreateVulkanDecodeDevice(&hw_device_ctx) >= 0) { + if (CreateSharedVulkanDevice(&hw_device_ctx) >= 0) { #else if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { #endif diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 2f0b6be8a..ec37404bc 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -30,6 +30,7 @@ #include "Exceptions.h" #include "Frame.h" #ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "GpuDevices.h" #include "VulkanFrameImage.h" #include "VulkanCompositor.h" #include "VulkanEncodeConverter.h" @@ -108,6 +109,7 @@ void FFmpegWriter::Open() { is_open = true; #ifdef OPENSHOT_ENABLE_GPU_FRAME gpu_export_failed = false; + gpu_export_logged = false; // FFmpeg binds a GPU frame to the first CUDA context it is sent to. Start // each export with fresh pools, so no frame is tied to an old encoder. ReleaseVulkanMemory(); @@ -1972,7 +1974,20 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { Logger::Instance()->AppendDebugMethod( "Encode Device not present, using default"); } - if (av_hwdevice_ctx_create(&hw_device_ctx, + // Encode on the card that renders, so VA-API can share its frames. That card + // may have no VA-API driver (e.g. NVIDIA in a hybrid laptop); then use the + // device chosen above. + bool device_created = false; +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__linux__) + if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VAAPI && VulkanRenderingEnabled()) { + const std::string render_node = ActiveGpuDevice().render_node; + device_created = !render_node.empty() && + av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type, render_node.c_str(), NULL, 0) >= 0; + if (!render_node.empty() && !device_created) + Logger::Instance()->Log("No VA-API on " + render_node + "; encoding on the default VA-API device", Logger::LevelWarning); + } +#endif + if (!device_created && av_hwdevice_ctx_create(&hw_device_ctx, hw_en_av_device_type, adapter_ptr, NULL, 0) < 0) { Logger::Instance()->AppendDebugMethod( @@ -2629,10 +2644,11 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { return; #if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL - // NVENC accepts encoder-owned CUDA surfaces. Preserve GPU residency through - // RGB->NV12 conversion and the Vulkan/CUDA device-to-device transfer. + // Keep frames on the GPU through RGB->NV12 conversion. NVENC takes a + // Vulkan/CUDA device-to-device copy; VA-API imports the NV12 images as DMA-BUF. + const bool vaapi_export = video_codec_ctx->pix_fmt == AV_PIX_FMT_VAAPI; if (!gpu_export_failed && hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx && - video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA) { + (video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA || vaapi_export)) { auto surface = frame->GetDeferredImage(); if (surface) { AVFrame* encoded = nullptr; @@ -2644,15 +2660,29 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { scale.scale(double(info.width) / src_w, double(info.height) / src_h); surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black); } - auto nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface)); + auto nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, vaapi_export)); encoded = av_frame_alloc(); if (!encoded) throw std::bad_alloc(); - int status = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); - if (status >= 0) status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0); - if (status < 0) throw std::runtime_error("Vulkan/CUDA export transfer failed: " + av_err2string(status)); + int status; + if (vaapi_export) { + // The mapped VA-API surface keeps the Vulkan image until the encoder is done + encoded->format = AV_PIX_FMT_VAAPI; + encoded->hw_frames_ctx = av_buffer_ref(video_codec_ctx->hw_frames_ctx); + if (!encoded->hw_frames_ctx) throw std::bad_alloc(); + status = av_hwframe_map(encoded, nv12->GetFrame(), AV_HWFRAME_MAP_READ); + if (status < 0) throw std::runtime_error("Vulkan/VA-API frame sharing failed: " + av_err2string(status)); + } else { + status = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); + if (status >= 0) status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0); + if (status < 0) throw std::runtime_error("Vulkan/CUDA export transfer failed: " + av_err2string(status)); + } av_frame_copy_props(encoded, nv12->GetFrame()); add_avframe(frame, encoded); - Logger::Instance()->AppendDebugMethod("Vulkan export GPU transfer to NVENC", "frame", frame->number); + if (!gpu_export_logged) { + gpu_export_logged = true; + Logger::Instance()->Log(vaapi_export ? "Vulkan export: sharing GPU frames with VA-API (no copy)" + : "Vulkan export: GPU transfer to NVENC", Logger::LevelInfo); + } return; } catch (const std::exception& error) { av_frame_free(&encoded); diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index bef75e7fe..94e82ae86 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -142,6 +142,7 @@ namespace openshot { int persistent_dst_size = 0; #ifdef OPENSHOT_ENABLE_GPU_FRAME bool gpu_export_failed = false; ///< Encode remaining frames from CPU pixels + bool gpu_export_logged = false; ///< The GPU export path was reported once #endif int audio_outbuf_size; diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp new file mode 100644 index 000000000..0ce5dcbca --- /dev/null +++ b/src/GpuDevices.cpp @@ -0,0 +1,276 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuDevices.h" +#include "Logger.h" +#include "VulkanFrameImage.h" +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +} +namespace openshot { +namespace { +// A temporary instance for listing devices; FFmpeg keeps its own for rendering. +struct ListingInstance { + VkInstance instance = VK_NULL_HANDLE; + ListingInstance() { + VkApplicationInfo app = {}; + app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo create = {}; + create.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + create.pApplicationInfo = &app; + if (vkCreateInstance(&create, nullptr, &instance) != VK_SUCCESS) + instance = VK_NULL_HANDLE; + } + ~ListingInstance() { if (instance) vkDestroyInstance(instance, nullptr); } + std::vector Devices() const { + uint32_t count = 0; + if (!instance) return {}; + vkEnumeratePhysicalDevices(instance, &count, nullptr); + std::vector devices(count); + vkEnumeratePhysicalDevices(instance, &count, devices.data()); + return devices; + } +}; + +bool HasExtension(VkPhysicalDevice device, const char* name) { + uint32_t count = 0; + vkEnumerateDeviceExtensionProperties(device, nullptr, &count, nullptr); + std::vector extensions(count); + vkEnumerateDeviceExtensionProperties(device, nullptr, &count, extensions.data()); + return std::any_of(extensions.begin(), extensions.end(), + [name](const VkExtensionProperties& e) { return std::strcmp(e.extensionName, name) == 0; }); +} + +GpuDevice Describe(VkPhysicalDevice device) { + VkPhysicalDeviceProperties basic; + vkGetPhysicalDeviceProperties(device, &basic); + VkPhysicalDeviceIDProperties ids = {}; + ids.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES; + VkPhysicalDeviceDriverProperties driver = {}; + driver.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES; + VkPhysicalDeviceDrmPropertiesEXT drm = {}; + drm.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT; + VkPhysicalDeviceProperties2 props = {}; + props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2; + props.pNext = &ids; + // Chain only the structures this device understands. + void** next = &ids.pNext; + const bool has_driver = basic.apiVersion >= VK_API_VERSION_1_2; + if (has_driver) { *next = &driver; next = &driver.pNext; } + const bool has_drm = HasExtension(device, VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME); + if (has_drm) { *next = &drm; next = &drm.pNext; } + vkGetPhysicalDeviceProperties2(device, &props); + + GpuDevice gpu; + gpu.name = basic.deviceName; + switch (basic.vendorID) { + case 0x10DE: gpu.vendor = "NVIDIA"; break; + case 0x1002: gpu.vendor = "AMD"; break; + case 0x8086: gpu.vendor = "Intel"; break; + default: { + char id[16]; + std::snprintf(id, sizeof(id), "0x%04x", basic.vendorID); + gpu.vendor = id; + } + } + switch (basic.deviceType) { + case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: gpu.type = "discrete"; break; + case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: gpu.type = "integrated"; break; + case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: gpu.type = "virtual"; break; + case VK_PHYSICAL_DEVICE_TYPE_CPU: gpu.type = "software"; break; + default: gpu.type = "other"; + } + if (has_driver) + gpu.driver = std::string(driver.driverName) + " " + driver.driverInfo; + for (uint8_t byte : ids.deviceUUID) { + char hex[3]; + std::snprintf(hex, sizeof(hex), "%02x", byte); + gpu.uuid += hex; + } + if (has_drm && drm.hasRender) + gpu.render_node = "/dev/dri/renderD" + std::to_string(drm.renderMinor); + + uint32_t families = 0; + vkGetPhysicalDeviceQueueFamilyProperties(device, &families, nullptr); + std::vector queues(families); + vkGetPhysicalDeviceQueueFamilyProperties(device, &families, queues.data()); + gpu.video_decode = std::any_of(queues.begin(), queues.end(), + [](const VkQueueFamilyProperties& q) { return q.queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR; }); + return gpu; +} + +// FFmpeg uses the first Vulkan device, which can be a software driver such as +// llvmpipe. List real GPUs instead, discrete before integrated. Software (CPU) +// devices are skipped: they cannot decode video and are slower than our CPU path. +std::vector VulkanDeviceCandidates() { + const char* chosen = std::getenv("OPENSHOT_VULKAN_DEVICE"); + if (chosen && chosen[0]) + return {chosen}; + ListingInstance listing; + if (!listing.instance) + return {""}; // Let FFmpeg report why Vulkan is unavailable + std::vector> ranked; + for (VkPhysicalDevice device : listing.Devices()) { + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(device, &props); + if (props.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU) + continue; + const int rank = props.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 0 + : props.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 1 : 2; + // Select by name: FFmpeg's device indices can be in a different order + ranked.emplace_back(rank, props.deviceName); + } + std::stable_sort(ranked.begin(), ranked.end(), + [](const auto& a, const auto& b) { return a.first < b.first; }); + std::vector candidates; + for (const auto& device : ranked) + candidates.push_back(device.second); + return candidates; +} + +VkPhysicalDevice PhysicalDevice(AVBufferRef* device) { + auto* context = reinterpret_cast(device->data); + return static_cast(context->hwctx)->phys_dev; +} + +// Try opening an encoder at a tiny size on the given hardware device. +bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format) { + const AVCodec* codec = avcodec_find_encoder_by_name(name); + if (!codec) return false; + AVCodecContext* context = avcodec_alloc_context3(codec); + if (!context) return false; + context->width = 256; + context->height = 256; + context->time_base = {1, 30}; + context->framerate = {30, 1}; + context->bit_rate = 1000000; + bool opened = false; + if (hw_format == AV_PIX_FMT_VAAPI) { + AVBufferRef* frames = av_hwframe_ctx_alloc(device); + if (frames) { + auto* fc = reinterpret_cast(frames->data); + fc->format = AV_PIX_FMT_VAAPI; + fc->sw_format = AV_PIX_FMT_NV12; + fc->width = context->width; + fc->height = context->height; + if (av_hwframe_ctx_init(frames) >= 0) { + context->pix_fmt = AV_PIX_FMT_VAAPI; + context->hw_frames_ctx = av_buffer_ref(frames); + } + av_buffer_unref(&frames); + } + } else { + // NVENC takes system-memory frames on the CUDA device given here + context->pix_fmt = AV_PIX_FMT_NV12; + context->hw_device_ctx = av_buffer_ref(device); + } + if (context->hw_frames_ctx || context->hw_device_ctx) + opened = avcodec_open2(context, codec, nullptr) >= 0; + avcodec_free_context(&context); + return opened; +} +} // namespace + +int CreateSharedVulkanDevice(AVBufferRef** output) { + // All readers share one device so their surfaces can be composed without interop. + // Keep an owning reference for the process lifetime; each caller gets its own ref. + struct SharedDevice { + std::mutex mutex; + AVBufferRef* device = nullptr; + bool tried = false; + ~SharedDevice() { av_buffer_unref(&device); } + }; + static SharedDevice shared; + const std::lock_guard lock(shared.mutex); + if (!shared.device) { + // Only search once; without a usable GPU every reader decodes on the CPU. + if (shared.tried) + return AVERROR(ENODEV); + shared.tried = true; + int result = AVERROR(ENODEV); + for (const auto& device : VulkanDeviceCandidates()) { + result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, + device.empty() ? nullptr : device.c_str(), nullptr, 0); + if (result >= 0) + break; + } + if (result < 0) { + Logger::Instance()->Log("No usable Vulkan GPU; decoding on the CPU", Logger::LevelWarning); + return result; + } + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(PhysicalDevice(shared.device), &props); + Logger::Instance()->Log(std::string("Vulkan device: ") + props.deviceName, Logger::LevelInfo); + } + *output = av_buffer_ref(shared.device); + return *output ? 0 : AVERROR(ENOMEM); +} + +GpuDevice ActiveGpuDevice() { + AVBufferRef* device = nullptr; + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&device) < 0) + return {}; + GpuDevice gpu = Describe(PhysicalDevice(device)); + gpu.active = true; + av_buffer_unref(&device); + return gpu; +} + +std::vector ListGpuDevices() { + const std::string active = ActiveGpuDevice().uuid; + ListingInstance listing; + std::vector devices; + for (VkPhysicalDevice device : listing.Devices()) { + devices.push_back(Describe(device)); + devices.back().active = !active.empty() && devices.back().uuid == active; + } + return devices; +} + +std::vector GpuEncoders() { + static std::mutex mutex; + static bool tested = false; + static std::vector encoders; + const std::lock_guard lock(mutex); + if (tested) + return encoders; + tested = true; + AVBufferRef* vulkan = nullptr; + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&vulkan) < 0) + return encoders; + const GpuDevice gpu = Describe(PhysicalDevice(vulkan)); + // Encoders that cannot open print errors; this is only a capability test. + const int log_level = av_log_get_level(); + av_log_set_level(AV_LOG_QUIET); + // CUDA derived from Vulkan is the same card (matched by UUID) + AVBufferRef* cuda = nullptr; + if (av_hwdevice_ctx_create_derived(&cuda, AV_HWDEVICE_TYPE_CUDA, vulkan, 0) >= 0) { + for (const char* name : {"h264_nvenc", "hevc_nvenc", "av1_nvenc"}) + if (EncoderOpens(name, cuda, AV_PIX_FMT_CUDA)) encoders.push_back(name); + av_buffer_unref(&cuda); + } + AVBufferRef* vaapi = nullptr; + if (!gpu.render_node.empty() && + av_hwdevice_ctx_create(&vaapi, AV_HWDEVICE_TYPE_VAAPI, gpu.render_node.c_str(), nullptr, 0) >= 0) { + for (const char* name : {"h264_vaapi", "hevc_vaapi", "av1_vaapi", "vp9_vaapi"}) + if (EncoderOpens(name, vaapi, AV_PIX_FMT_VAAPI)) encoders.push_back(name); + av_buffer_unref(&vaapi); + } + av_log_set_level(log_level); + av_buffer_unref(&vulkan); + std::string list; + for (const auto& name : encoders) + list += (list.empty() ? "" : ", ") + name; + Logger::Instance()->Log("GPU encoders on " + gpu.name + ": " + (list.empty() ? "none" : list), Logger::LevelInfo); + return encoders; +} +} diff --git a/src/GpuDevices.h b/src/GpuDevices.h new file mode 100644 index 000000000..d85c0fd9a --- /dev/null +++ b/src/GpuDevices.h @@ -0,0 +1,37 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_GPU_DEVICES_H +#define OPENSHOT_GPU_DEVICES_H +#include +#include +#ifndef SWIG +struct AVBufferRef; +#endif +namespace openshot { +/// A Vulkan graphics card as OpenShot sees it. +struct GpuDevice { + std::string name; + std::string vendor; ///< "NVIDIA", "AMD", "Intel", or the PCI vendor id + std::string type; ///< "discrete", "integrated", "virtual", "software" or "other" + std::string driver; ///< Driver name and version, when the driver reports them + std::string uuid; ///< Stable id (hex), the same one CUDA reports for the card + std::string render_node; ///< Linux DRM render node such as /dev/dri/renderD128, if known + bool video_decode = false; ///< Has a Vulkan video decode queue + bool active = false; ///< The card OpenShot renders with +}; +/// Every Vulkan device on this system, including software ones. Empty without Vulkan. +std::vector ListGpuDevices(); +/// The card OpenShot renders with, choosing it on first use. Empty name when +/// Vulkan is off (OPENSHOT_RENDER_BACKEND=cpu) or no usable GPU was found. +GpuDevice ActiveGpuDevice(); +/// Hardware encoders (e.g. "h264_nvenc", "hevc_vaapi") that open on the active +/// card. Tested once per process with a tiny frame size; takes a fraction of a second. +std::vector GpuEncoders(); +#ifndef SWIG +/// The shared Vulkan device every reader decodes on, created on first use with the +/// best GPU (OPENSHOT_VULKAN_DEVICE, an index or a name, overrides it). Returns a new +/// reference, or a negative FFmpeg error when there is no usable GPU. +int CreateSharedVulkanDevice(AVBufferRef** output); +#endif +} +#endif diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 4443bd7fa..d395c4bb4 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -34,15 +34,81 @@ constexpr int kSlots=4; // its last frame is released. struct OutputPools { std::mutex mutex; - std::map,AVBufferRef*> pools; + std::map,AVBufferRef*> pools; ~OutputPools(){ for(auto& item:pools) av_buffer_unref(&item.second); } }; OutputPools& SharedPools(){ static OutputPools pools; return pools; } +constexpr VkImageUsageFlags kOutputUsage=VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT; +// Shareable NV12 frames for VA-API: linear images in memory exported as DMA-BUF. +// FFmpeg 7.1 only exports opaque FDs (enough for CUDA), so these come from our pool. +const uint64_t kLinearModifier=0; // DRM_FORMAT_MOD_LINEAR, importable by every VA-API driver +VkImageDrmFormatModifierListCreateInfoEXT kLinearModifiers{VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT,nullptr,1,&kLinearModifier}; +struct ShareablePool { AVVulkanDeviceContext* vk; int width,height; std::vector families; }; +void FreeShareableFrame(void* opaque,uint8_t* data){ + auto* pool=static_cast(opaque); + auto* f=reinterpret_cast(data); + VkDevice device=pool->vk->act_dev; + for(int i=0;i<2;++i){ + // Encoders and mappings may still use the image; wait like FFmpeg does + if(f->sem[i]){ + VkSemaphoreWaitInfo wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + wait.semaphoreCount=1;wait.pSemaphores=&f->sem[i];wait.pValues=&f->sem_value[i]; + vkWaitSemaphores(device,&wait,UINT64_MAX); + vkDestroySemaphore(device,f->sem[i],nullptr); + } + if(f->img[i]) vkDestroyImage(device,f->img[i],nullptr); + if(f->mem[i]) vkFreeMemory(device,f->mem[i],nullptr); + } + av_freep(&f->internal); + av_free(f); +} +AVBufferRef* AllocShareableFrame(void* opaque,size_t){ + auto* pool=static_cast(opaque); + AVVkFrame* f=av_vk_frame_alloc(); + if(!f) return nullptr; + AVBufferRef* buffer=av_buffer_create(reinterpret_cast(f),sizeof(AVVkFrame),FreeShareableFrame,pool,0); + if(!buffer){ av_freep(&f->internal); av_free(f); return nullptr; } + VkDevice device=pool->vk->act_dev; + VkPhysicalDeviceMemoryProperties memory; + vkGetPhysicalDeviceMemoryProperties(pool->vk->phys_dev,&memory); + f->tiling=VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; + for(int i=0;i<2;++i){ + VkExternalMemoryImageCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,&kLinearModifiers,VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; + VkImageCreateInfo ci{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,&external}; + ci.imageType=VK_IMAGE_TYPE_2D;ci.format=i?VK_FORMAT_R8G8_UNORM:VK_FORMAT_R8_UNORM; + ci.extent={uint32_t(i?pool->width/2:pool->width),uint32_t(i?pool->height/2:pool->height),1}; + ci.mipLevels=1;ci.arrayLayers=1;ci.samples=VK_SAMPLE_COUNT_1_BIT;ci.tiling=VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; + ci.usage=kOutputUsage;ci.initialLayout=VK_IMAGE_LAYOUT_UNDEFINED; + ci.sharingMode=pool->families.size()>1?VK_SHARING_MODE_CONCURRENT:VK_SHARING_MODE_EXCLUSIVE; + ci.queueFamilyIndexCount=pool->families.size()>1?pool->families.size():0;ci.pQueueFamilyIndices=pool->families.data(); + VkMemoryRequirements req; + uint32_t type=UINT32_MAX; + if(vkCreateImage(device,&ci,nullptr,&f->img[i])!=VK_SUCCESS){ av_buffer_unref(&buffer); return nullptr; } + vkGetImageMemoryRequirements(device,f->img[i],&req); + for(uint32_t t=0;timg[i]}; + VkExportMemoryAllocateInfo exported{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,&dedicated,VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; + VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,&exported,req.size,type}; + VkSemaphoreTypeCreateInfo timeline{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,nullptr,VK_SEMAPHORE_TYPE_TIMELINE,0}; + VkSemaphoreCreateInfo si{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,&timeline}; + if(type==UINT32_MAX || vkAllocateMemory(device,&ai,nullptr,&f->mem[i])!=VK_SUCCESS || + vkBindImageMemory(device,f->img[i],f->mem[i],0)!=VK_SUCCESS || + vkCreateSemaphore(device,&si,nullptr,&f->sem[i])!=VK_SUCCESS){ av_buffer_unref(&buffer); return nullptr; } + f->size[i]=req.size;f->layout[i]=VK_IMAGE_LAYOUT_UNDEFINED;f->queue_family[i]=VK_QUEUE_FAMILY_IGNORED; + } + return buffer; +} +void FreeShareablePool(void* opaque){ delete static_cast(opaque); } +bool HasDeviceExtension(const AVVulkanDeviceContext* vk,const char* name){ + for(int i=0;inb_enabled_dev_extensions;++i) if(std::string(vk->enabled_dev_extensions[i])==name) return true; + return false; +} // Returns a new reference the caller must release. -AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ +AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12,bool shareable){ auto& shared=SharedPools(); const std::lock_guard lock(shared.mutex); - auto& pool=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12}]; + auto& pool=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12,shareable}]; if(!pool){ AVBufferRef* frames=av_hwframe_ctx_alloc(device_ref); if(!frames) throw std::bad_alloc(); @@ -51,9 +117,30 @@ AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ fc->width=w; fc->height=h; auto* vc=reinterpret_cast(fc->hwctx); if(encode_nv12) vc->flags=AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; - vc->usage=static_cast(VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT); + vc->usage=static_cast(kOutputUsage); + if(shareable){ + auto* vk=reinterpret_cast(reinterpret_cast(device_ref->data)->hwctx); + if(!encode_nv12 || !HasDeviceExtension(vk,VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME) || + !HasDeviceExtension(vk,VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)){ + av_buffer_unref(&frames); shared.pools.erase({reinterpret_cast(device_ref->data),w,h,encode_nv12,shareable}); + throw std::runtime_error("Vulkan device cannot export DMA-BUF images"); + } + auto* opaque=new ShareablePool{vk,w,h,{}}; + for(int i=0;inb_qf;++i) + if(std::find(opaque->families.begin(),opaque->families.end(),uint32_t(vk->qf[i].idx))==opaque->families.end()) + opaque->families.push_back(vk->qf[i].idx); + vc->tiling=VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; + vc->create_pnext=&kLinearModifiers; + fc->pool=av_buffer_pool_init2(sizeof(AVVkFrame),opaque,AllocShareableFrame,FreeShareablePool); + if(!fc->pool){ delete opaque; av_buffer_unref(&frames); throw std::bad_alloc(); } + // FFmpeg leaves a caller's pool to the caller; free it while the device lives + fc->free=[](AVHWFramesContext* context){ av_buffer_pool_uninit(&context->pool); }; + } const int status=av_hwframe_ctx_init(frames); - if(status<0){ av_buffer_unref(&frames); AvCheck(status,"Initialize Vulkan compositor output"); } + if(status<0){ + av_buffer_unref(&frames); shared.pools.erase({reinterpret_cast(device_ref->data),w,h,encode_nv12,shareable}); + AvCheck(status,"Initialize Vulkan compositor output"); + } pool=frames; } AVBufferRef* ref=av_buffer_ref(pool); @@ -103,8 +190,9 @@ class Pipeline { int next_slot=0; int width,height; bool encode_nv12; + bool shareable; public: - Pipeline(AVBufferRef* device_owner,int w,int h,bool encode=false):width(w),height(h),encode_nv12(encode) { + Pipeline(AVBufferRef* device_owner,int w,int h,bool encode=false,bool share=false):width(w),height(h),encode_nv12(encode),shareable(share) { device_ref=av_buffer_ref(device_owner); if(!device_ref) throw std::bad_alloc(); device_context=reinterpret_cast(device_ref->data); @@ -206,7 +294,7 @@ class Pipeline { } FramePtr output(av_frame_alloc()); if(!output) throw std::bad_alloc(); - AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12); + AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12,shareable); const int allocated=av_hwframe_get_buffer(pool,output.get(),0); av_buffer_unref(&pool); AvCheck(allocated,"Allocate compositor output"); @@ -366,18 +454,18 @@ std::shared_ptr RenderVulkanImage(std::shared_ptrsecond.pipeline; return pipeline->Render(*src,back.get(),matrix,opacity,clear,high_quality); } -std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source) { +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, bool shareable) { auto src=std::dynamic_pointer_cast(source); if(!src) throw std::invalid_argument("NV12 conversion requires a Vulkan surface"); const auto* frame=src->GetFrame(); if(frame->width%2 || frame->height%2) throw std::invalid_argument("NV12 export requires even dimensions"); auto* fc=reinterpret_cast(frame->hw_frames_ctx->data); - using Key=std::tuple; + using Key=std::tuple; thread_local std::unique_ptr pipeline; thread_local Key key{}; - Key requested{fc->device_ctx,frame->width,frame->height}; + Key requested{fc->device_ctx,frame->width,frame->height,shareable}; if(!pipeline || key!=requested){ - pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true); + pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true,shareable); key=requested; } return pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black); diff --git a/src/VulkanEncodeConverter.h b/src/VulkanEncodeConverter.h index 232464a59..5f364f0fc 100644 --- a/src/VulkanEncodeConverter.h +++ b/src/VulkanEncodeConverter.h @@ -6,6 +6,7 @@ #include namespace openshot { /// Convert an RGBA Vulkan surface to encoder-ready limited-range NV12 on the GPU. -std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source); +/// shareable uses linear DMA-BUF images that VA-API can import without a copy. +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, bool shareable = false); } #endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3abed91ed..ce956b844 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -186,6 +186,8 @@ foreach(tname ${OPENSHOT_TESTS}) ) endif() if(tname MATCHES "^Vulkan") + # Same Vulkan headers as the library (newer than some system packages) + target_include_directories(openshot-${tname}-test PRIVATE ${Vulkan_INCLUDE_DIR}) # Report missing GPU, display or interop support as skipped, not passed. list(APPEND test_properties SKIP_REGULAR_EXPRESSION "Skipping Vulkan test:" diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 034c8a5c2..986898c6a 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -4,6 +4,7 @@ #include "VulkanCompositor.h" #include "VulkanFrameImage.h" #include "VulkanEncodeConverter.h" +#include "GpuDevices.h" #include "FFmpegReader.h" #include "Clip.h" #include "DummyReader.h" @@ -11,7 +12,10 @@ #include #include extern "C" { +#include #include +#include +#include #include } namespace { @@ -26,7 +30,7 @@ bool VulkanAvailable() { return false; } AVBufferRef* device=nullptr; - if(av_hwdevice_ctx_create(&device,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)<0) { + if(openshot::CreateSharedVulkanDevice(&device)<0) { WARN("Skipping Vulkan test: no Vulkan device is available"); return false; } @@ -48,10 +52,84 @@ class PatternReader : public openshot::DummyReader { } }; } +TEST_CASE("Vulkan GPU listing reports the card OpenShot renders with", "[vulkan][hardware]") { + if(!VulkanAvailable()) return; + const auto active=openshot::ActiveGpuDevice(); + REQUIRE(!active.name.empty()); + CHECK(active.active); + CHECK(active.type!="software"); + CHECK(active.uuid.size()==32); + int active_count=0; + for(const auto& gpu : openshot::ListGpuDevices()) { + CHECK(!gpu.name.empty()); + if(gpu.active) { ++active_count; CHECK(gpu.uuid==active.uuid); } + } + CHECK(active_count==1); + // Every reported encoder exists in this FFmpeg build; the list is cached. + const auto encoders=openshot::GpuEncoders(); + for(const auto& name : encoders) CHECK(avcodec_find_encoder_by_name(name.c_str())); + CHECK(openshot::GpuEncoders()==encoders); +} +TEST_CASE("Vulkan shareable NV12 export frames match and export as DMA-BUF", "[vulkan][hardware]") { + if(!VulkanAvailable()) return; + AVBufferRef* device_ptr=nullptr; + REQUIRE(openshot::CreateSharedVulkanDevice(&device_ptr)>=0); + Buffer device(device_ptr); + Buffer pool(av_hwframe_ctx_alloc(device.get())); + REQUIRE(pool); + auto* context=reinterpret_cast(pool->data); + context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; + context->width=32;context->height=32; + REQUIRE(av_hwframe_ctx_init(pool.get())>=0); + Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); + cpu->format=AV_PIX_FMT_RGBA;cpu->width=32;cpu->height=32; + REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); + for(int y=0;y<32;++y)for(int x=0;x<32;++x){ + auto* p=cpu->data[0]+y*cpu->linesize[0]+x*4; + p[0]=255;p[1]=0;p[2]=0;p[3]=255; + } + REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); + REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); + auto red=std::make_shared(gpu.get()); + std::shared_ptr shared; + try { + shared=openshot::ConvertVulkanToNV12(red,true); + } catch(const std::exception& error) { + WARN("Skipping Vulkan test: no DMA-BUF export on this device: " << error.what()); + return; + } + // Same pixels as the regular NV12 pool, in linear images. + auto surface=std::dynamic_pointer_cast(shared); + REQUIRE(surface); + CHECK(reinterpret_cast(surface->GetFrame()->data[0])->tiling==VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT); + Frame downloaded(av_frame_alloc()); + REQUIRE(av_hwframe_transfer_data(downloaded.get(),surface->GetFrame(),0)>=0); + REQUIRE(downloaded->format==AV_PIX_FMT_NV12); + const int y=downloaded->data[0][4*downloaded->linesize[0]+4]; + const int u=downloaded->data[1][2*downloaded->linesize[1]+4]; + const int v=downloaded->data[1][2*downloaded->linesize[1]+5]; + CHECK(y>=80);CHECK(y<=83); + CHECK(u>=89);CHECK(u<=92); + CHECK(v>=238);CHECK(v<=241); + // The frames map to DMA-BUF (Y and UV planes, linear) for VA-API. + Frame drm(av_frame_alloc()); + drm->format=AV_PIX_FMT_DRM_PRIME; + const int mapped=av_hwframe_map(drm.get(),surface->GetFrame(),AV_HWFRAME_MAP_READ); + if(mapped==AVERROR(ENOSYS)) { + WARN("FFmpeg was built without libdrm; DMA-BUF mapping not checked"); + return; + } + REQUIRE(mapped>=0); + const auto* desc=reinterpret_cast(drm->data[0]); + CHECK(desc->nb_objects==2); + CHECK(desc->nb_layers==2); + CHECK(desc->objects[0].format_modifier==0); + CHECK(desc->objects[0].fd>=0); +} TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan][hardware]") { if(!VulkanAvailable()) return; AVBufferRef* device_ptr=nullptr; - REQUIRE(av_hwdevice_ctx_create(&device_ptr,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)>=0); + REQUIRE(openshot::CreateSharedVulkanDevice(&device_ptr)>=0); Buffer device(device_ptr); Buffer pool(av_hwframe_ctx_alloc(device.get())); REQUIRE(pool); @@ -197,7 +275,7 @@ TEST_CASE("Vulkan sequential decoding preserves pending audio video frames", "[v TEST_CASE("Vulkan high quality reader resizing uses bicubic antialiasing", "[vulkan][hardware][quality]") { if(!VulkanAvailable()) return; AVBufferRef* raw_device=nullptr; - REQUIRE(av_hwdevice_ctx_create(&raw_device,AV_HWDEVICE_TYPE_VULKAN,nullptr,nullptr,0)>=0); + REQUIRE(openshot::CreateSharedVulkanDevice(&raw_device)>=0); Buffer device(raw_device),pool(av_hwframe_ctx_alloc(raw_device)); REQUIRE(pool); auto* context=reinterpret_cast(pool->data); diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index b066d829d..51a21d6db 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -3,6 +3,7 @@ #include "openshot_catch.h" #include "Frame.h" #include "VulkanFrameImage.h" +#include "GpuDevices.h" #include "Qt/VideoRenderer.h" #include "Qt/VulkanOpenGLPresenter.h" #include @@ -53,7 +54,7 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ return; } Frames data; - if (av_hwdevice_ctx_create(&data.device, AV_HWDEVICE_TYPE_VULKAN, nullptr, nullptr, 0) < 0) { + if (openshot::CreateSharedVulkanDevice(&data.device) < 0) { WARN("Skipping Vulkan test: no Vulkan device is available"); return; } From 5b969761d5c6e108c6f8934fb1c1bb021ab325ad Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Oct 2026 13:31:06 -0500 Subject: [PATCH 14/50] Encode Vulkan frames without CPU readback Write native Vulkan NV12 directly into encoder-owned frames, with CPU fallback for unavailable encoders and incompatible surfaces. Keep encoder state per writer and preserve NVENC's separate-plane CUDA transfer. Add copy-free and fallback regression tests, readback diagnostics, and GPU compilation in the FFmpeg compatibility matrix. Verified on the Radeon 780M with FFmpeg 6.1 and 7.1.3; NVIDIA hardware testing remains outstanding. --- .github/workflows/ffmpeg-compatibility.yml | 16 +- examples/TimelineExportBenchmark.cpp | 19 +- src/FFmpegReader.cpp | 6 +- src/FFmpegWriter.cpp | 137 +++++++++----- src/FFmpegWriter.h | 11 +- src/GpuDevices.cpp | 10 +- src/VulkanCompositor.cpp | 147 +++++---------- src/VulkanEncodeConverter.h | 5 +- src/VulkanFrameImage.cpp | 4 + src/VulkanFrameImage.h | 3 + tests/VulkanCompositor.cpp | 199 ++++++++++++++++----- 11 files changed, 346 insertions(+), 211 deletions(-) diff --git a/.github/workflows/ffmpeg-compatibility.yml b/.github/workflows/ffmpeg-compatibility.yml index 4443fa0f3..078298474 100644 --- a/.github/workflows/ffmpeg-compatibility.yml +++ b/.github/workflows/ffmpeg-compatibility.yml @@ -3,10 +3,16 @@ name: FFmpeg compatibility on: + push: + branches: [vulkan-mvp] pull_request: paths: - 'src/FFmpeg*' + - 'src/Vulkan*' + - 'src/GpuDevices*' + - 'src/Qt/Vulkan*' - 'tests/FFmpeg*' + - 'tests/Vulkan*' - 'tests/CMakeLists.txt' - '.github/workflows/ffmpeg-compatibility.yml' workflow_dispatch: @@ -57,7 +63,7 @@ jobs: sudo apt-get update sudo apt-get install -y cmake ninja-build pkg-config nasm catch2 \ libasound2-dev qtbase5-dev libqt5svg5-dev libbabl-dev libjsoncpp-dev \ - libx264-dev libvpx-dev libvorbis-dev + libx264-dev libvpx-dev libvorbis-dev libvulkan-dev glslang-tools - name: Build isolated FFmpeg and audio dependencies run: | prefix="$RUNNER_TEMP/ffmpeg-dependencies" @@ -83,9 +89,11 @@ jobs: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH="$RUNNER_TEMP/ffmpeg-dependencies" \ -DENABLE_LIB_DOCS=OFF -DENABLE_PYTHON=OFF -DENABLE_OPENCV=OFF \ - -DENABLE_MAGICK=OFF -DUSE_QT6=OFF + -DENABLE_MAGICK=OFF -DUSE_QT6=OFF -DENABLE_GPU_FRAME=ON cmake --build build --parallel 2 --target \ - openshot-FFmpegCapabilities-test openshot-FFmpegWriter-test + openshot-FFmpegCapabilities-test openshot-FFmpegWriter-test \ + openshot-VulkanCompositor-test openshot-VulkanPresentation-test - name: Test capability validation and audio/video exports working-directory: build - run: ctest -L '^(FFmpegCapabilities|FFmpegWriter)$' --output-on-failure --timeout 180 -j2 + # Hardware cases skip without Vulkan support; the CPU fallback still runs. + run: ctest -L '^(FFmpegCapabilities|FFmpegWriter|VulkanCompositor|VulkanPresentation)$' --output-on-failure --timeout 180 -j2 diff --git a/examples/TimelineExportBenchmark.cpp b/examples/TimelineExportBenchmark.cpp index c61e80b2d..3739de5ba 100644 --- a/examples/TimelineExportBenchmark.cpp +++ b/examples/TimelineExportBenchmark.cpp @@ -3,6 +3,9 @@ #include "Clip.h" #include "Timeline.h" #include "FFmpegWriter.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanFrameImage.h" +#endif #include #include #include @@ -11,8 +14,8 @@ // Identical full-resolution rendering + NVENC workload for CPU/GPU builds. // Optional audio input exercises timeline mixing and muxing. Output is replaced. int main(int argc,char** argv) { - if(argc<3 || argc>6){ - std::cerr<<"Usage: openshot-export-benchmark VIDEO OUTPUT.mp4 [FRAMES=120] [LAYERS=2] [AUDIO]\n"; + if(argc<3 || argc>7){ + std::cerr<<"Usage: openshot-export-benchmark VIDEO OUTPUT.mp4 [FRAMES=120] [LAYERS=2] [AUDIO|-] [CODEC=h264_nvenc]\n"; return 2; } try { @@ -34,7 +37,7 @@ int main(int argc,char** argv) { } timeline.AddClip(clip.get());clips.push_back(std::move(clip)); } - if(argc>5){ + if(argc>5 && std::string(argv[5])!="-"){ auto audio=std::make_unique(argv[5]);audio->Layer(-1); // Do not render cover art embedded in an audio file. audio->has_video=openshot::Keyframe(0.0); @@ -42,11 +45,14 @@ int main(int argc,char** argv) { } timeline.Open(); openshot::FFmpegWriter writer(argv[2]); - writer.SetVideoOptions(true,"h264_nvenc",openshot::Fraction(24,1),3840,2160,openshot::Fraction(1,1),false,false,40000000); + writer.SetVideoOptions(true,argc>6?argv[6]:"h264_nvenc",openshot::Fraction(24,1),3840,2160,openshot::Fraction(1,1),false,false,40000000); writer.SetAudioOptions(true,"aac",48000,2,openshot::LAYOUT_STEREO,192000); writer.Open(); int deferred=0; std::shared_ptr last; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const auto readbacks=openshot::VulkanFrameReadbackCount(); +#endif const auto start=std::chrono::steady_clock::now(); for(int n=1;n<=count;++n){ last=timeline.GetFrame(n); @@ -57,7 +63,10 @@ int main(int argc,char** argv) { } writer.Close(); const double seconds=std::chrono::duration(std::chrono::steady_clock::now()-start).count(); - std::cout<<"frames="<thread_count = std::min(FF_VIDEO_NUM_PROCESSORS, 16); - if (pCodec == NULL) { - throw InvalidCodec("A valid video codec could not be found for this file.", path); - } - // Init options av_dict_set(&opts, "strict", "experimental", 0); #if USE_HW_ACCEL diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index ec37404bc..f89d7dc58 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -41,18 +41,27 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +bool IsVulkanEncoder(const std::string& name) { + return name == "h264_vulkan" || name == "hevc_vulkan" || name == "av1_vulkan"; +} +std::string SoftwareEncoder(const std::string& name) { + const std::vector candidates = name == "h264_vulkan" ? std::vector{"libx264", "libopenh264"} + : name == "hevc_vulkan" ? std::vector{"libx265"} + : std::vector{"libsvtav1", "libaom-av1", "librav1e"}; + for (const auto& candidate : candidates) + if (avcodec_find_encoder_by_name(candidate.c_str())) return candidate; + return {}; +} +} +#endif + // Multiplexer parameters temporary storage AVDictionary *mux_dict = NULL; #if USE_HW_ACCEL -int hw_en_on = 1; // Is set in UI -int hw_en_supported = 0; // Is set by FFmpegWriter -AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE; -AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_VAAPI; -static AVBufferRef *hw_device_ctx = NULL; -AVFrame *hw_frame = NULL; - -static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height) +static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int64_t width, int64_t height, AVPixelFormat format) { AVBufferRef *hw_frames_ref; AVHWFramesContext *frames_ctx = NULL; @@ -63,7 +72,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6 return -1; } frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data); - frames_ctx->format = hw_en_av_pix_fmt; + frames_ctx->format = format; frames_ctx->sw_format = AV_PIX_FMT_NV12; frames_ctx->width = width; frames_ctx->height = height; @@ -120,8 +129,26 @@ void FFmpegWriter::Open() { PrepareStreams(); // Now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers - if (info.has_video && video_st) + if (info.has_video && video_st) { +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL + try { + open_video(oc, video_st); + } catch (const std::exception& error) { + // Choose the replacement before the muxer header is written. + // Switching encoders after packets are emitted is unsafe. + const auto fallback = IsVulkanEncoder(info.vcodec) ? SoftwareEncoder(info.vcodec) : std::string(); + if (fallback.empty() || write_header) throw; + Logger::Instance()->Log("Vulkan encoder CPU fallback: " + std::string(error.what()) + "; using " + fallback, Logger::LevelWarning); + av_buffer_unref(&hw_device_ctx); + SetVideoOptions(true, fallback, info.fps, info.width, info.height, info.pixel_ratio, + info.interlaced_frame, info.top_field_first, info.video_bit_rate); + add_video_stream(video_st); + open_video(oc, video_st); + } +#else open_video(oc, video_st); +#endif + } if (info.has_audio && audio_st) open_audio(oc, audio_st); @@ -180,11 +207,29 @@ void FFmpegWriter::initialize_streams() { // Set video export options void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (IsVulkanEncoder(codec) && (!USE_HW_ACCEL || !VulkanRenderingEnabled() || !avcodec_find_encoder_by_name(codec.c_str()))) { + const auto fallback = SoftwareEncoder(codec); + if (!fallback.empty()) { + Logger::Instance()->Log("Vulkan encoder unavailable; using CPU encoder " + fallback, Logger::LevelWarning); + codec = fallback; + } + } +#endif // Set the video options if (codec.length() > 0) { const AVCodec *new_codec; // Check if the codec selected is a hardware accelerated codec #if USE_HW_ACCEL +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (IsVulkanEncoder(codec)) { + new_codec = avcodec_find_encoder_by_name(codec.c_str()); + hw_en_on = hw_en_supported = true; + hw_en_av_pix_fmt = AV_PIX_FMT_VULKAN; + hw_en_av_device_type = AV_HWDEVICE_TYPE_VULKAN; + } else +#endif + { #if defined(__linux__) if (strstr(codec.c_str(), "_vaapi") != NULL) { new_codec = avcodec_find_encoder_by_name(codec.c_str()); @@ -236,6 +281,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction f #else // unknown OS new_codec = avcodec_find_encoder_by_name(codec.c_str()); #endif //__linux__/_WIN32/__APPLE__ + } #else // USE_HW_ACCEL new_codec = avcodec_find_encoder_by_name(codec.c_str()); #endif // USE_HW_ACCEL @@ -1660,7 +1706,7 @@ AVStream *FFmpegWriter::add_audio_stream() { } // Add a video output stream -AVStream *FFmpegWriter::add_video_stream() { +AVStream *FFmpegWriter::add_video_stream(AVStream* existing) { // Find the video codec const AVCodec *codec = avcodec_find_encoder_by_name(info.vcodec.c_str()); if (codec == NULL) @@ -1672,7 +1718,7 @@ AVStream *FFmpegWriter::add_video_stream() { } // Create a new video stream - AVStream* st = avformat_new_stream(oc, codec); + AVStream* st = existing ? existing : avformat_new_stream(oc, codec); if (!st) throw OutOfMemory("Could not allocate memory for the video stream.", path); @@ -1692,9 +1738,11 @@ AVStream *FFmpegWriter::add_video_stream() { c->sample_aspect_ratio.den = info.pixel_ratio.den; /* Init video encoder options */ + // VA-API AV1 supports bitrate control; the legacy AV1 quality conversion + // below clears the bitrate and uses a qp option this encoder does not expose. if (info.video_bit_rate >= 1000 #if (LIBAVCODEC_VERSION_MAJOR >= 58) - && c->codec_id != AV_CODEC_ID_AV1 + && (c->codec_id != AV_CODEC_ID_AV1 || info.vcodec == "av1_vaapi" || info.vcodec == "av1_vulkan") #endif ) { c->bit_rate = info.video_bit_rate; @@ -1978,6 +2026,13 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { // may have no VA-API driver (e.g. NVIDIA in a hybrid laptop); then use the // device chosen above. bool device_created = false; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VULKAN) { + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&hw_device_ctx) < 0) + throw InvalidCodec("No Vulkan rendering device is available", path); + device_created = true; + } +#endif #if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__linux__) if (hw_en_av_device_type == AV_HWDEVICE_TYPE_VAAPI && VulkanRenderingEnabled()) { const std::string render_node = ActiveGpuDevice().render_node; @@ -2042,10 +2097,12 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { switch (video_codec_ctx->codec_id) { case AV_CODEC_ID_H264: video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames - video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE; - av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0); - av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0); - av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN); + if (hw_en_av_pix_fmt != AV_PIX_FMT_VULKAN) { + video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE; + av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0); + av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0); + av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN); + } break; case AV_CODEC_ID_HEVC: // tested to work with defaults @@ -2062,7 +2119,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { // set hw_frames_ctx for encoder's AVCodecContext int err; - if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height)) < 0) + if ((err = set_hwframe_ctx(video_codec_ctx, hw_device_ctx, info.width, info.height, hw_en_av_pix_fmt)) < 0) { Logger::Instance()->AppendDebugMethod( "FFmpegWriter::open_video (set_hwframe_ctx) ERROR faled to set hwframe context", @@ -2644,11 +2701,11 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { return; #if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL - // Keep frames on the GPU through RGB->NV12 conversion. NVENC takes a - // Vulkan/CUDA device-to-device copy; VA-API imports the NV12 images as DMA-BUF. - const bool vaapi_export = video_codec_ctx->pix_fmt == AV_PIX_FMT_VAAPI; + // Native Vulkan writes directly into encoder-owned NV12. NVENC retains its + // device-to-device transfer path; other encoders use the established CPU path. + const bool native_vulkan = video_codec_ctx->pix_fmt == AV_PIX_FMT_VULKAN; if (!gpu_export_failed && hw_en_on && hw_en_supported && video_codec_ctx->hw_frames_ctx && - (video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA || vaapi_export)) { + (native_vulkan || video_codec_ctx->pix_fmt == AV_PIX_FMT_CUDA)) { auto surface = frame->GetDeferredImage(); if (surface) { AVFrame* encoded = nullptr; @@ -2660,27 +2717,27 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { scale.scale(double(info.width) / src_w, double(info.height) / src_h); surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black); } - auto nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, vaapi_export)); + // NVENC keeps its existing separate-plane conversion and GPU + // transfer. Native Vulkan writes into the encoder's own frame. + std::shared_ptr nv12; + if (!native_vulkan) + nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface)); encoded = av_frame_alloc(); if (!encoded) throw std::bad_alloc(); - int status; - if (vaapi_export) { - // The mapped VA-API surface keeps the Vulkan image until the encoder is done - encoded->format = AV_PIX_FMT_VAAPI; - encoded->hw_frames_ctx = av_buffer_ref(video_codec_ctx->hw_frames_ctx); - if (!encoded->hw_frames_ctx) throw std::bad_alloc(); - status = av_hwframe_map(encoded, nv12->GetFrame(), AV_HWFRAME_MAP_READ); - if (status < 0) throw std::runtime_error("Vulkan/VA-API frame sharing failed: " + av_err2string(status)); + const int allocated = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); + if (allocated < 0) throw std::runtime_error("Allocate encoder frame: " + av_err2string(allocated)); + if (native_vulkan) { + nv12 = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, encoded)); } else { - status = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); - if (status >= 0) status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0); + const int status = av_hwframe_transfer_data(encoded, nv12->GetFrame(), 0); if (status < 0) throw std::runtime_error("Vulkan/CUDA export transfer failed: " + av_err2string(status)); } + av_frame_copy_props(encoded, nv12->GetFrame()); add_avframe(frame, encoded); if (!gpu_export_logged) { gpu_export_logged = true; - Logger::Instance()->Log(vaapi_export ? "Vulkan export: sharing GPU frames with VA-API (no copy)" + Logger::Instance()->Log(native_vulkan ? "Vulkan export: encoder-owned GPU frames (no copy)" : "Vulkan export: GPU transfer to NVENC", Logger::LevelInfo); } return; @@ -2890,17 +2947,15 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra #endif ) { if (!(hw_frame = av_frame_alloc())) { - std::clog << "Error code: av_hwframe_alloc\n"; + throw OutOfMemory("Could not allocate hardware upload frame", path); } if (av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, hw_frame, 0) < 0) { - std::clog << "Error code: av_hwframe_get_buffer\n"; - } - if (!hw_frame->hw_frames_ctx) { - std::clog << "Error hw_frames_ctx.\n"; + av_frame_free(&hw_frame); + throw ErrorEncodingVideo("Could not allocate hardware upload buffer", -1); } - hw_frame->format = AV_PIX_FMT_NV12; if ( av_hwframe_transfer_data(hw_frame, frame_final, 0) < 0) { - std::clog << "Error while transferring frame data to surface.\n"; + av_frame_free(&hw_frame); + throw ErrorEncodingVideo("Could not upload CPU frame to encoder", -1); } av_frame_copy_props(hw_frame, frame_final); } diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index 94e82ae86..fa9603464 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -136,6 +136,15 @@ namespace openshot { uint8_t *audio_outbuf; uint8_t *audio_encoder_buffer; +#if USE_HW_ACCEL + // Hardware encoder ownership belongs to this export, never another writer. + bool hw_en_on = false; + bool hw_en_supported = false; + AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE; + AVHWDeviceType hw_en_av_device_type = AV_HWDEVICE_TYPE_NONE; + AVBufferRef* hw_device_ctx = nullptr; + AVFrame* hw_frame = nullptr; +#endif AVFrame *persistent_src_frame = nullptr; AVFrame *persistent_dst_frame = nullptr; uint8_t *persistent_dst_buffer = nullptr; @@ -167,7 +176,7 @@ namespace openshot { AVStream *add_audio_stream(); /// Add a video output stream - AVStream *add_video_stream(); + AVStream *add_video_stream(AVStream* existing = nullptr); /// Allocate an AVFrame object AVFrame *allocate_avframe(PixelFormat pix_fmt, int width, int height, int *buffer_size, uint8_t *new_buffer); diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 0ce5dcbca..6366646cc 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -154,16 +154,16 @@ bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format context->framerate = {30, 1}; context->bit_rate = 1000000; bool opened = false; - if (hw_format == AV_PIX_FMT_VAAPI) { + if (hw_format == AV_PIX_FMT_VAAPI || hw_format == AV_PIX_FMT_VULKAN) { AVBufferRef* frames = av_hwframe_ctx_alloc(device); if (frames) { auto* fc = reinterpret_cast(frames->data); - fc->format = AV_PIX_FMT_VAAPI; + fc->format = hw_format; fc->sw_format = AV_PIX_FMT_NV12; fc->width = context->width; fc->height = context->height; if (av_hwframe_ctx_init(frames) >= 0) { - context->pix_fmt = AV_PIX_FMT_VAAPI; + context->pix_fmt = hw_format; context->hw_frames_ctx = av_buffer_ref(frames); } av_buffer_unref(&frames); @@ -251,6 +251,10 @@ std::vector GpuEncoders() { // Encoders that cannot open print errors; this is only a capability test. const int log_level = av_log_get_level(); av_log_set_level(AV_LOG_QUIET); + // Prefer the same Vulkan device/API for every vendor, when its driver and + // this FFmpeg build support the requested codec. Probe, never assume. + for (const char* name : {"h264_vulkan", "hevc_vulkan", "av1_vulkan"}) + if (EncoderOpens(name, vulkan, AV_PIX_FMT_VULKAN)) encoders.push_back(name); // CUDA derived from Vulkan is the same card (matched by UUID) AVBufferRef* cuda = nullptr; if (av_hwdevice_ctx_create_derived(&cuda, AV_HWDEVICE_TYPE_CUDA, vulkan, 0) >= 0) { diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index d395c4bb4..292a0f27e 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -34,81 +34,16 @@ constexpr int kSlots=4; // its last frame is released. struct OutputPools { std::mutex mutex; - std::map,AVBufferRef*> pools; + std::map,AVBufferRef*> pools; ~OutputPools(){ for(auto& item:pools) av_buffer_unref(&item.second); } }; OutputPools& SharedPools(){ static OutputPools pools; return pools; } constexpr VkImageUsageFlags kOutputUsage=VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT; -// Shareable NV12 frames for VA-API: linear images in memory exported as DMA-BUF. -// FFmpeg 7.1 only exports opaque FDs (enough for CUDA), so these come from our pool. -const uint64_t kLinearModifier=0; // DRM_FORMAT_MOD_LINEAR, importable by every VA-API driver -VkImageDrmFormatModifierListCreateInfoEXT kLinearModifiers{VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT,nullptr,1,&kLinearModifier}; -struct ShareablePool { AVVulkanDeviceContext* vk; int width,height; std::vector families; }; -void FreeShareableFrame(void* opaque,uint8_t* data){ - auto* pool=static_cast(opaque); - auto* f=reinterpret_cast(data); - VkDevice device=pool->vk->act_dev; - for(int i=0;i<2;++i){ - // Encoders and mappings may still use the image; wait like FFmpeg does - if(f->sem[i]){ - VkSemaphoreWaitInfo wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; - wait.semaphoreCount=1;wait.pSemaphores=&f->sem[i];wait.pValues=&f->sem_value[i]; - vkWaitSemaphores(device,&wait,UINT64_MAX); - vkDestroySemaphore(device,f->sem[i],nullptr); - } - if(f->img[i]) vkDestroyImage(device,f->img[i],nullptr); - if(f->mem[i]) vkFreeMemory(device,f->mem[i],nullptr); - } - av_freep(&f->internal); - av_free(f); -} -AVBufferRef* AllocShareableFrame(void* opaque,size_t){ - auto* pool=static_cast(opaque); - AVVkFrame* f=av_vk_frame_alloc(); - if(!f) return nullptr; - AVBufferRef* buffer=av_buffer_create(reinterpret_cast(f),sizeof(AVVkFrame),FreeShareableFrame,pool,0); - if(!buffer){ av_freep(&f->internal); av_free(f); return nullptr; } - VkDevice device=pool->vk->act_dev; - VkPhysicalDeviceMemoryProperties memory; - vkGetPhysicalDeviceMemoryProperties(pool->vk->phys_dev,&memory); - f->tiling=VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; - for(int i=0;i<2;++i){ - VkExternalMemoryImageCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,&kLinearModifiers,VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; - VkImageCreateInfo ci{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,&external}; - ci.imageType=VK_IMAGE_TYPE_2D;ci.format=i?VK_FORMAT_R8G8_UNORM:VK_FORMAT_R8_UNORM; - ci.extent={uint32_t(i?pool->width/2:pool->width),uint32_t(i?pool->height/2:pool->height),1}; - ci.mipLevels=1;ci.arrayLayers=1;ci.samples=VK_SAMPLE_COUNT_1_BIT;ci.tiling=VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; - ci.usage=kOutputUsage;ci.initialLayout=VK_IMAGE_LAYOUT_UNDEFINED; - ci.sharingMode=pool->families.size()>1?VK_SHARING_MODE_CONCURRENT:VK_SHARING_MODE_EXCLUSIVE; - ci.queueFamilyIndexCount=pool->families.size()>1?pool->families.size():0;ci.pQueueFamilyIndices=pool->families.data(); - VkMemoryRequirements req; - uint32_t type=UINT32_MAX; - if(vkCreateImage(device,&ci,nullptr,&f->img[i])!=VK_SUCCESS){ av_buffer_unref(&buffer); return nullptr; } - vkGetImageMemoryRequirements(device,f->img[i],&req); - for(uint32_t t=0;timg[i]}; - VkExportMemoryAllocateInfo exported{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,&dedicated,VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; - VkMemoryAllocateInfo ai{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,&exported,req.size,type}; - VkSemaphoreTypeCreateInfo timeline{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,nullptr,VK_SEMAPHORE_TYPE_TIMELINE,0}; - VkSemaphoreCreateInfo si{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,&timeline}; - if(type==UINT32_MAX || vkAllocateMemory(device,&ai,nullptr,&f->mem[i])!=VK_SUCCESS || - vkBindImageMemory(device,f->img[i],f->mem[i],0)!=VK_SUCCESS || - vkCreateSemaphore(device,&si,nullptr,&f->sem[i])!=VK_SUCCESS){ av_buffer_unref(&buffer); return nullptr; } - f->size[i]=req.size;f->layout[i]=VK_IMAGE_LAYOUT_UNDEFINED;f->queue_family[i]=VK_QUEUE_FAMILY_IGNORED; - } - return buffer; -} -void FreeShareablePool(void* opaque){ delete static_cast(opaque); } -bool HasDeviceExtension(const AVVulkanDeviceContext* vk,const char* name){ - for(int i=0;inb_enabled_dev_extensions;++i) if(std::string(vk->enabled_dev_extensions[i])==name) return true; - return false; -} // Returns a new reference the caller must release. -AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12,bool shareable){ +AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ auto& shared=SharedPools(); const std::lock_guard lock(shared.mutex); - auto& pool=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12,shareable}]; + auto& pool=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12}]; if(!pool){ AVBufferRef* frames=av_hwframe_ctx_alloc(device_ref); if(!frames) throw std::bad_alloc(); @@ -118,27 +53,9 @@ AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12,boo auto* vc=reinterpret_cast(fc->hwctx); if(encode_nv12) vc->flags=AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; vc->usage=static_cast(kOutputUsage); - if(shareable){ - auto* vk=reinterpret_cast(reinterpret_cast(device_ref->data)->hwctx); - if(!encode_nv12 || !HasDeviceExtension(vk,VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME) || - !HasDeviceExtension(vk,VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME)){ - av_buffer_unref(&frames); shared.pools.erase({reinterpret_cast(device_ref->data),w,h,encode_nv12,shareable}); - throw std::runtime_error("Vulkan device cannot export DMA-BUF images"); - } - auto* opaque=new ShareablePool{vk,w,h,{}}; - for(int i=0;inb_qf;++i) - if(std::find(opaque->families.begin(),opaque->families.end(),uint32_t(vk->qf[i].idx))==opaque->families.end()) - opaque->families.push_back(vk->qf[i].idx); - vc->tiling=VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT; - vc->create_pnext=&kLinearModifiers; - fc->pool=av_buffer_pool_init2(sizeof(AVVkFrame),opaque,AllocShareableFrame,FreeShareablePool); - if(!fc->pool){ delete opaque; av_buffer_unref(&frames); throw std::bad_alloc(); } - // FFmpeg leaves a caller's pool to the caller; free it while the device lives - fc->free=[](AVHWFramesContext* context){ av_buffer_pool_uninit(&context->pool); }; - } const int status=av_hwframe_ctx_init(frames); if(status<0){ - av_buffer_unref(&frames); shared.pools.erase({reinterpret_cast(device_ref->data),w,h,encode_nv12,shareable}); + av_buffer_unref(&frames); shared.pools.erase({reinterpret_cast(device_ref->data),w,h,encode_nv12}); AvCheck(status,"Initialize Vulkan compositor output"); } pool=frames; @@ -176,23 +93,22 @@ class Pipeline { VkPipelineLayout layout=VK_NULL_HANDLE; VkPipeline pipeline=VK_NULL_HANDLE; VkSampler sampler=VK_NULL_HANDLE; - // One in-flight submission. Consumers order themselves through the frames' - // timeline semaphores, and FFmpeg waits on them before freeing a frame, so - // the slot only keeps its own commands and views until it retires. + // Consumers order submissions through the frames' timeline semaphores. + // Keep encoder-owned targets alive until their views and work are retired. struct Slot { VkDescriptorSet set=VK_NULL_HANDLE; VkCommandBuffer cmd=VK_NULL_HANDLE; VkFence fence=VK_NULL_HANDLE; bool busy=false; std::vector views; + FramePtr target; }; std::array ring; int next_slot=0; int width,height; bool encode_nv12; - bool shareable; public: - Pipeline(AVBufferRef* device_owner,int w,int h,bool encode=false,bool share=false):width(w),height(h),encode_nv12(encode),shareable(share) { + Pipeline(AVBufferRef* device_owner,int w,int h,bool encode=false):width(w),height(h),encode_nv12(encode) { device_ref=av_buffer_ref(device_owner); if(!device_ref) throw std::bad_alloc(); device_context=reinterpret_cast(device_ref->data); @@ -272,6 +188,7 @@ class Pipeline { } for(auto view:slot.views) vkDestroyImageView(device,view,nullptr); slot.views.clear(); + slot.target.reset(); } VkImageView View(VkImage image,VkFormat format,VkImageAspectFlags aspect){ VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; @@ -282,7 +199,7 @@ class Pipeline { return view; } std::shared_ptr Render(const VulkanFrameImage& input,const VulkanFrameImage* background, - const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false) { + const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false,const AVFrame* target=nullptr) { const auto* src=input.GetFrame(); const auto* src_fc=reinterpret_cast(src->hw_frames_ctx->data); if(src_fc->device_ctx!=device_context) throw std::runtime_error("Vulkan source device mismatch"); @@ -294,10 +211,22 @@ class Pipeline { } FramePtr output(av_frame_alloc()); if(!output) throw std::bad_alloc(); - AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12,shareable); - const int allocated=av_hwframe_get_buffer(pool,output.get(),0); - av_buffer_unref(&pool); - AvCheck(allocated,"Allocate compositor output"); + if(target){ + auto* target_fc=target->hw_frames_ctx?reinterpret_cast(target->hw_frames_ctx->data):nullptr; + if(!encode_nv12 || !target_fc || target->format!=AV_PIX_FMT_VULKAN || + target_fc->sw_format!=AV_PIX_FMT_NV12 || target_fc->device_ctx!=device_context || + target->width!=width || target->height!=height) + throw std::invalid_argument("Encoder target must be matching NV12 on the rendering device"); + const auto* target_vk=static_cast(target_fc->hwctx); + if(!(target_vk->usage&VK_IMAGE_USAGE_STORAGE_BIT)) + throw std::runtime_error("Encoder target cannot be written by a Vulkan shader"); + AvCheck(av_frame_ref(output.get(),target),"Retain encoder target"); + }else{ + AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12); + const int allocated=av_hwframe_get_buffer(pool,output.get(),0); + av_buffer_unref(&pool); + AvCheck(allocated,"Allocate compositor output"); + } AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); output->colorspace=encode_nv12?AVCOL_SPC_SMPTE170M:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; if(encode_nv12 && src_fc->sw_format!=AV_PIX_FMT_RGBA) throw std::runtime_error("Encoder conversion requires RGBA"); @@ -344,6 +273,12 @@ class Pipeline { VkCommandBuffer cmd=slot.cmd; const VkDescriptorSet set=slot.set; try { + // Ordinary outputs use our shared pools. An encoder's pool can + // disappear when its writer closes, before this cached pipeline. + if(target){ + slot.target.reset(av_frame_clone(output.get())); + if(!slot.target) throw std::bad_alloc(); + } std::array views{}; auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ auto view=View(image,format,aspect);owned_views.push_back(view);return view; @@ -358,9 +293,9 @@ class Pipeline { } views[3]=back?make_view(reinterpret_cast(back->data[0])->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT):views[0]; if(encode_nv12){ - if(!of->img[1] || of->img[1]==of->img[0]) throw std::runtime_error("NV12 encoder output requires separate plane images"); - views[3]=make_view(of->img[0],VK_FORMAT_R8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); - views[4]=make_view(of->img[1],VK_FORMAT_R8G8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); + const bool separate=of->img[1] && of->img[1]!=of->img[0]; + views[3]=make_view(of->img[0],VK_FORMAT_R8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); + views[4]=make_view(of->img[separate?1:0],VK_FORMAT_R8G8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); }else views[4]=make_view(of->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); std::array images{}; std::array writes{}; @@ -378,7 +313,7 @@ class Pipeline { std::vector barriers; for(auto* f:all){ auto* vf=reinterpret_cast(f->data[0]); - const int count=f==src?plane_count:(f==output.get() && encode_nv12?2:1); + const int count=f==src?plane_count:(f==output.get() && encode_nv12 && of->img[1] && of->img[1]!=of->img[0]?2:1); for(int i=0;i(vf->access[i]); @@ -454,21 +389,21 @@ std::shared_ptr RenderVulkanImage(std::shared_ptrsecond.pipeline; return pipeline->Render(*src,back.get(),matrix,opacity,clear,high_quality); } -std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, bool shareable) { +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, const AVFrame* target) { auto src=std::dynamic_pointer_cast(source); if(!src) throw std::invalid_argument("NV12 conversion requires a Vulkan surface"); const auto* frame=src->GetFrame(); if(frame->width%2 || frame->height%2) throw std::invalid_argument("NV12 export requires even dimensions"); auto* fc=reinterpret_cast(frame->hw_frames_ctx->data); - using Key=std::tuple; + using Key=std::tuple; thread_local std::unique_ptr pipeline; thread_local Key key{}; - Key requested{fc->device_ctx,frame->width,frame->height,shareable}; + Key requested{fc->device_ctx,frame->width,frame->height}; if(!pipeline || key!=requested){ - pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true,shareable); + pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true); key=requested; } - return pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black); + return pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black,false,target); } void ReleaseVulkanMemory() { diff --git a/src/VulkanEncodeConverter.h b/src/VulkanEncodeConverter.h index 5f364f0fc..cb36612f4 100644 --- a/src/VulkanEncodeConverter.h +++ b/src/VulkanEncodeConverter.h @@ -4,9 +4,10 @@ #define OPENSHOT_VULKAN_ENCODE_CONVERTER_H #include "FrameImage.h" #include +struct AVFrame; namespace openshot { /// Convert an RGBA Vulkan surface to encoder-ready limited-range NV12 on the GPU. -/// shareable uses linear DMA-BUF images that VA-API can import without a copy. -std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, bool shareable = false); +/// An optional encoder-owned Vulkan NV12 target is written directly, without a copy. +std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, const AVFrame* target = nullptr); } #endif diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 6ed7ecf35..03cf7ef6b 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -1,6 +1,7 @@ // Copyright (c) 2026 OpenShot Studios, LLC // SPDX-License-Identifier: LGPL-3.0-or-later #include "VulkanFrameImage.h" +#include #include "VulkanCompositor.h" #include #include @@ -40,6 +41,8 @@ void Validate(const AVFrame* frame) { throw std::invalid_argument("Expected an FFmpeg Vulkan hardware frame"); } } +namespace { std::atomic readback_count{0}; } +uint64_t VulkanFrameReadbackCount() { return readback_count.load(std::memory_order_relaxed); } bool VulkanRenderingEnabled() { const char* backend = std::getenv("OPENSHOT_RENDER_BACKEND"); return !(backend && std::strcmp(backend, "cpu") == 0); @@ -81,6 +84,7 @@ QImage VulkanFrameImage::Materialize() const { std::lock_guard lock(impl->mutex); auto cpu = AllocateFrame(); Check(av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0), "Read back Vulkan frame"); + readback_count.fetch_add(1,std::memory_order_relaxed); Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); QImage result(GetWidth(), GetHeight(), QImage::Format_RGBA8888); if (result.isNull()) throw std::bad_alloc(); diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h index b133fd42c..94d12f5cb 100644 --- a/src/VulkanFrameImage.h +++ b/src/VulkanFrameImage.h @@ -3,11 +3,14 @@ #ifndef OPENSHOT_VULKAN_FRAME_IMAGE_H #define OPENSHOT_VULKAN_FRAME_IMAGE_H #include "FrameImage.h" +#include #include struct AVFrame; namespace openshot { /// False when OPENSHOT_RENDER_BACKEND=cpu selects the CPU pipeline at runtime. bool VulkanRenderingEnabled(); +/// Process-wide count of GPU images materialized as CPU pixels (diagnostics). +uint64_t VulkanFrameReadbackCount(); /// A retained FFmpeg Vulkan image. CPU pixels are created only by Materialize(). class VulkanFrameImage final : public FrameImage { public: diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 986898c6a..591102910 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -6,7 +6,11 @@ #include "VulkanEncodeConverter.h" #include "GpuDevices.h" #include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include +#include #include "Clip.h" +#include "Timeline.h" #include "DummyReader.h" #include #include @@ -14,7 +18,6 @@ extern "C" { #include #include -#include #include #include } @@ -70,61 +73,162 @@ TEST_CASE("Vulkan GPU listing reports the card OpenShot renders with", "[vulkan] for(const auto& name : encoders) CHECK(avcodec_find_encoder_by_name(name.c_str())); CHECK(openshot::GpuEncoders()==encoders); } -TEST_CASE("Vulkan shareable NV12 export frames match and export as DMA-BUF", "[vulkan][hardware]") { +TEST_CASE("VA-API AV1 export accepts a requested bitrate", "[vulkan][hardware][writer]") { if(!VulkanAvailable()) return; - AVBufferRef* device_ptr=nullptr; - REQUIRE(openshot::CreateSharedVulkanDevice(&device_ptr)>=0); - Buffer device(device_ptr); - Buffer pool(av_hwframe_ctx_alloc(device.get())); + const auto encoders=openshot::GpuEncoders(); + if(std::find(encoders.begin(),encoders.end(),"av1_vaapi")==encoders.end()){ + WARN("Skipping Vulkan test: no VA-API AV1 encoder"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const auto path=directory.filePath("av1.mp4").toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true,"av1_vaapi",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + REQUIRE_NOTHROW(writer.Open()); + for(int n=1;n<=3;++n) writer.WriteFrame(std::make_shared(n,256,256,"red")); + writer.Close(); + openshot::FFmpegReader reader(path); + reader.Open(); + const auto* decoder=avcodec_find_decoder_by_name(reader.info.vcodec.c_str()); + REQUIRE(decoder); + CHECK(decoder->id==AV_CODEC_ID_AV1); + auto image=reader.GetFrame(1)->GetImage(); + REQUIRE(image); + CHECK(image->pixelColor(128,128).red()>200); + CHECK(image->pixelColor(128,128).blue()<30); + reader.Close(); +} +TEST_CASE("Native Vulkan export keeps decoded and composited frames on the GPU", "[vulkan][hardware][writer]") { + if(!VulkanAvailable()) return; + const auto encoders=openshot::GpuEncoders(); + if(std::find(encoders.begin(),encoders.end(),"h264_vulkan")==encoders.end()){ + WARN("Skipping Vulkan test: no native Vulkan H.264 encoder"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + // Repeat exports to exercise encoder surface reuse and owner cleanup. + for(int pass=0;pass<2;++pass){ + const int width=pass?640:256,height=pass?360:256; + openshot::FFmpegReader base_reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + openshot::FFmpegReader overlay_reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + openshot::Clip base(&base_reader),overlay(&overlay_reader); + openshot::Timeline timeline(width,height,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.SetMaxSize(width,height); + base.Layer(0);overlay.Layer(1); + overlay.scale_x=openshot::Keyframe(0.7);overlay.rotation=openshot::Keyframe(12.0); + overlay.alpha=openshot::Keyframe(0.6); + timeline.AddClip(&base);timeline.AddClip(&overlay);timeline.Open(); + const auto path=directory.filePath(QString("native-%1.mp4").arg(pass)).toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),width,height,openshot::Fraction(1,1),false,false,1000000); + writer.Open(); + REQUIRE(writer.info.vcodec=="h264_vulkan"); // CPU fallback must not pass this test. + const auto readbacks=openshot::VulkanFrameReadbackCount(); + for(int n=1;n<=24;++n){ + auto frame=timeline.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + writer.WriteFrame(frame); + REQUIRE(frame->GetDeferredImage()); + } + writer.Close(); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + CHECK(base_reader.HardwareDecodeSuccessful()); + CHECK(overlay_reader.HardwareDecodeSuccessful()); + timeline.Close(); + // Decode the finished bitstream on the CPU only after measuring export. + openshot::FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.width==width);CHECK(reader.info.height==height); + CHECK(reader.info.video_length==24); + auto image=reader.GetFrame(12)->GetImage(); + REQUIRE(image);CHECK(image->width()==width);CHECK(image->height()==height); + reader.Close(); + } +} +TEST_CASE("Vulkan export accepts CPU frames and keeps writers independent", "[vulkan][writer][fallback]") { + // Run this with OPENSHOT_RENDER_BACKEND=cpu, an invalid Vulkan device, or + // an older FFmpeg too: the requested codec must fall back before the header. + if(!avcodec_find_encoder_by_name("libx264")){ + WARN("Skipping Vulkan test: CPU fallback test requires libx264"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const auto gpu_path=directory.filePath("gpu-or-fallback.mp4").toStdString(); + const auto cpu_path=directory.filePath("cpu.mp4").toStdString(); + openshot::FFmpegWriter gpu(gpu_path),cpu(cpu_path); + gpu.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + cpu.SetVideoOptions(true,"libx264",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + gpu.Open();cpu.Open(); + if(!openshot::VulkanRenderingEnabled()) CHECK(gpu.info.vcodec!="h264_vulkan"); + for(int n=1;n<=3;++n){ + gpu.WriteFrame(std::make_shared(n,256,256,"red")); + cpu.WriteFrame(std::make_shared(n,256,256,"blue")); + } + gpu.Close();cpu.Close(); + for(const auto& item:std::vector>{{gpu_path,Qt::red},{cpu_path,Qt::blue}}){ + openshot::FFmpegReader reader(item.first); + reader.Open(); + CHECK(reader.info.video_length==3); + auto image=reader.GetFrame(2)->GetImage(); + REQUIRE(image); + const auto actual=image->pixelColor(128,128); + CHECK(std::abs(actual.red()-item.second.red())<25); + CHECK(std::abs(actual.blue()-item.second.blue())<25); + reader.Close(); + } +} +TEST_CASE("Vulkan export recovers from an incompatible GPU surface", "[vulkan][hardware][fallback]") { + if(!VulkanAvailable()) return; + const auto encoders=openshot::GpuEncoders(); + if(std::find(encoders.begin(),encoders.end(),"h264_vulkan")==encoders.end()){ + WARN("Skipping Vulkan test: no native Vulkan H.264 encoder"); + return; + } + // A different logical device cannot write into the encoder's pool. The + // original frame is still readable, so this failure can use CPU pixels. + AVBufferRef* other=nullptr; + const auto name=openshot::ActiveGpuDevice().name; + REQUIRE(av_hwdevice_ctx_create(&other,AV_HWDEVICE_TYPE_VULKAN,name.c_str(),nullptr,0)>=0); + Buffer device(other),pool(av_hwframe_ctx_alloc(other)); REQUIRE(pool); auto* context=reinterpret_cast(pool->data); context->format=AV_PIX_FMT_VULKAN;context->sw_format=AV_PIX_FMT_RGBA; - context->width=32;context->height=32; + context->width=256;context->height=256; REQUIRE(av_hwframe_ctx_init(pool.get())>=0); Frame cpu(av_frame_alloc()),gpu(av_frame_alloc()); - cpu->format=AV_PIX_FMT_RGBA;cpu->width=32;cpu->height=32; + cpu->format=AV_PIX_FMT_RGBA;cpu->width=256;cpu->height=256; REQUIRE(av_frame_get_buffer(cpu.get(),0)>=0); - for(int y=0;y<32;++y)for(int x=0;x<32;++x){ - auto* p=cpu->data[0]+y*cpu->linesize[0]+x*4; - p[0]=255;p[1]=0;p[2]=0;p[3]=255; + for(int y=0;y<256;++y)for(int x=0;x<256;++x){ + auto* pixel=cpu->data[0]+y*cpu->linesize[0]+x*4; + pixel[0]=255;pixel[1]=0;pixel[2]=0;pixel[3]=255; } REQUIRE(av_hwframe_get_buffer(pool.get(),gpu.get(),0)>=0); REQUIRE(av_hwframe_transfer_data(gpu.get(),cpu.get(),0)>=0); - auto red=std::make_shared(gpu.get()); - std::shared_ptr shared; - try { - shared=openshot::ConvertVulkanToNV12(red,true); - } catch(const std::exception& error) { - WARN("Skipping Vulkan test: no DMA-BUF export on this device: " << error.what()); - return; + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const auto path=directory.filePath("fallback.mp4").toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),256,256,openshot::Fraction(1,1),false,false,1000000); + writer.Open(); + const auto before=openshot::VulkanFrameReadbackCount(); + for(int n=1;n<=3;++n){ + auto frame=std::make_shared(n,256,256,"black"); + frame->AddDeferredImage(std::make_shared(gpu.get())); + REQUIRE_NOTHROW(writer.WriteFrame(frame)); + CHECK_FALSE(frame->GetDeferredImage()); } - // Same pixels as the regular NV12 pool, in linear images. - auto surface=std::dynamic_pointer_cast(shared); - REQUIRE(surface); - CHECK(reinterpret_cast(surface->GetFrame()->data[0])->tiling==VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT); - Frame downloaded(av_frame_alloc()); - REQUIRE(av_hwframe_transfer_data(downloaded.get(),surface->GetFrame(),0)>=0); - REQUIRE(downloaded->format==AV_PIX_FMT_NV12); - const int y=downloaded->data[0][4*downloaded->linesize[0]+4]; - const int u=downloaded->data[1][2*downloaded->linesize[1]+4]; - const int v=downloaded->data[1][2*downloaded->linesize[1]+5]; - CHECK(y>=80);CHECK(y<=83); - CHECK(u>=89);CHECK(u<=92); - CHECK(v>=238);CHECK(v<=241); - // The frames map to DMA-BUF (Y and UV planes, linear) for VA-API. - Frame drm(av_frame_alloc()); - drm->format=AV_PIX_FMT_DRM_PRIME; - const int mapped=av_hwframe_map(drm.get(),surface->GetFrame(),AV_HWFRAME_MAP_READ); - if(mapped==AVERROR(ENOSYS)) { - WARN("FFmpeg was built without libdrm; DMA-BUF mapping not checked"); - return; - } - REQUIRE(mapped>=0); - const auto* desc=reinterpret_cast(drm->data[0]); - CHECK(desc->nb_objects==2); - CHECK(desc->nb_layers==2); - CHECK(desc->objects[0].format_modifier==0); - CHECK(desc->objects[0].fd>=0); + writer.Close(); + CHECK(openshot::VulkanFrameReadbackCount()==before+3); + openshot::FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.video_length==3); + const auto image=reader.GetFrame(2)->GetImage(); + REQUIRE(image);CHECK(image->pixelColor(128,128).red()>220); + CHECK(image->pixelColor(128,128).blue()<25); + reader.Close(); } TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan][hardware]") { if(!VulkanAvailable()) return; @@ -167,6 +271,11 @@ TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan] auto encoded = openshot::ConvertVulkanToNV12(second); auto encoded_surface = std::dynamic_pointer_cast(encoded); REQUIRE(encoded_surface); + // NVENC still receives separate Y/UV Vulkan images for CUDA interop. + const auto* planes=reinterpret_cast(encoded_surface->GetFrame()->data[0]); + REQUIRE(planes->img[0]!=VK_NULL_HANDLE); + CHECK(planes->img[1]!=VK_NULL_HANDLE); + CHECK(planes->img[0]!=planes->img[1]); Frame downloaded(av_frame_alloc()); REQUIRE(downloaded); REQUIRE(av_hwframe_transfer_data(downloaded.get(), encoded_surface->GetFrame(), 0) >= 0); From 86669a1c7497497a4e2ba66cb27da5490660947f Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 29 Oct 2026 18:18:30 -0500 Subject: [PATCH 15/50] Skip native Vulkan encoders on old Vulkan loaders Ubuntu 20.04's loader (1.2.131) hands FFmpeg a stub without a driver function for the video encode queries, and opening h264_vulkan crashes (FFmpeg's own CLI crashes the same way). Probe the native Vulkan encoders only with loader 1.3.270 or newer, the oldest verified to fail cleanly. Use h264_vulkan/hevc_vulkan only when the probe opened them, falling back to the CPU encoder otherwise, instead of trusting that FFmpeg includes them. Verified on an RTX 5070 (driver 580) with loaders 1.2.131 and 1.3.270: Vulkan tests pass, NVENC exports stay on the GPU, and the Vulkan presets fall back to libx264/libx265. --- src/FFmpegWriter.cpp | 4 +++- src/GpuDevices.cpp | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index f89d7dc58..cf7d31592 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -208,7 +208,9 @@ void FFmpegWriter::initialize_streams() { // Set video export options void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction fps, int width, int height, Fraction pixel_ratio, bool interlaced, bool top_field_first, int bit_rate) { #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (IsVulkanEncoder(codec) && (!USE_HW_ACCEL || !VulkanRenderingEnabled() || !avcodec_find_encoder_by_name(codec.c_str()))) { + // Only encoders the probe opened are safe; some drivers and loaders crash. + const auto gpu_encoders = IsVulkanEncoder(codec) ? GpuEncoders() : std::vector(); + if (IsVulkanEncoder(codec) && (!USE_HW_ACCEL || std::find(gpu_encoders.begin(), gpu_encoders.end(), codec) == gpu_encoders.end())) { const auto fallback = SoftwareEncoder(codec); if (!fallback.empty()) { Logger::Instance()->Log("Vulkan encoder unavailable; using CPU encoder " + fallback, Logger::LevelWarning); diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 6366646cc..e6969b908 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -253,8 +253,15 @@ std::vector GpuEncoders() { av_log_set_level(AV_LOG_QUIET); // Prefer the same Vulkan device/API for every vendor, when its driver and // this FFmpeg build support the requested codec. Probe, never assume. - for (const char* name : {"h264_vulkan", "hevc_vulkan", "av1_vulkan"}) - if (EncoderOpens(name, vulkan, AV_PIX_FMT_VULKAN)) encoders.push_back(name); + // Old loaders (1.2.131 on Ubuntu 20.04) return a stub without a driver + // function for the encode queries, and FFmpeg crashes calling it. + // 1.3.270 is the oldest loader verified to fail cleanly instead. + uint32_t loader_version = 0; + if (vkEnumerateInstanceVersion(&loader_version) == VK_SUCCESS && + loader_version >= VK_MAKE_API_VERSION(0, 1, 3, 270)) { + for (const char* name : {"h264_vulkan", "hevc_vulkan", "av1_vulkan"}) + if (EncoderOpens(name, vulkan, AV_PIX_FMT_VULKAN)) encoders.push_back(name); + } // CUDA derived from Vulkan is the same card (matched by UUID) AVBufferRef* cuda = nullptr; if (av_hwdevice_ctx_create_derived(&cuda, AV_HWDEVICE_TYPE_CUDA, vulkan, 0) >= 0) { From 30d2abe6d049a6b0c76f0a6bdc4192837303bae6 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 30 Oct 2026 11:53:55 -0500 Subject: [PATCH 16/50] Use FFmpeg 7.1 in runner builds and installer packaging --- .gitlab-ci.yml | 17 +++++++++++------ tests/CMakeLists.txt | 7 +++++-- tests/FFmpegReader.cpp | 5 +++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d620419a1..c4249b0e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,7 +49,7 @@ linux-builder: - export FFMPEGDIR="$FFMPEG_ROOT" - mkdir -p build; cd build; - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support" - - cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ + - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ - cmake --build . --parallel $(nproc) # Some focal libraries link the system FFmpeg 4.2; never load two FFmpegs - ldd src/libopenshot.so | grep -E 'libav|libsw|vulkan' @@ -74,7 +74,7 @@ mac-builder: variables: OPENCV_ROOT: /usr/local/openshot/opencv-4.13.0 OpenCV_DIR: /usr/local/openshot/opencv-4.13.0/lib/cmake/opencv4 - FFMPEGDIR: /usr/local/openshot/ffmpeg-4.4 + FFMPEGDIR: /usr/local/openshot/ffmpeg-7.1.3 artifacts: expire_in: 6 months paths: @@ -90,7 +90,7 @@ mac-builder: - export PKG_CONFIG_PATH="$FFMPEGDIR/lib/pkgconfig:$PKG_CONFIG_PATH" - export DYLD_LIBRARY_PATH=$FFMPEGDIR/lib:$OPENCV_ROOT/lib:$DYLD_LIBRARY_PATH - mkdir -p build; cd build; - - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ + - cmake -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ - cmake --build . --parallel $(sysctl -n hw.ncpu) - make install - find "$CI_PROJECT_DIR/build/install-x64" -type f \( -name "*.dylib" -or -name "*.so" \) -exec sh -c 'for f do if ! otool -l "$f" | grep -q "$OPENCV_ROOT/lib"; then install_name_tool -add_rpath "$OPENCV_ROOT/lib" "$f" || true; fi; done' sh {} + @@ -120,9 +120,11 @@ windows-builder-x64: - $env:OpenCV_DIR = "$env:OPENCV_ROOT\lib\cmake\opencv4" - $env:Path = "$env:OPENCV_ROOT\x64\mingw\bin;$env:OPENCV_ROOT\bin;C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; - $env:MSYSTEM = "MINGW64" + - $env:FFMPEGDIR = "C:/msys64/mingw64" + - if ((& "$env:FFMPEGDIR/bin/ffmpeg.exe" -version | Select-Object -First 1) -notmatch "^ffmpeg version 7[.]1[.]") { throw "FFmpeg 7.1 is required" } - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -DENABLE_GPU_FRAME=ON - cmake --build build --parallel $([Environment]::ProcessorCount) - ctest --test-dir build --output-on-failure -VV - cmake --install build @@ -150,7 +152,7 @@ windows-builder-arm64: - "if ($CI_COMMIT_REF_NAME -eq \"arm64-support\" -and -not (Select-String -Path \"build/install-arm64/share/libopenshot-audio.env\" -Pattern \"^CI_COMMIT_REF_NAME:arm64-support$\" -Quiet)) { throw \"Matching arm64-support audio artifacts are required\" }" - $env:MSYSTEM = "CLANGARM64" - $env:Path = "C:\msys64\clangarm64\bin;C:\msys64\usr\bin;" + $env:Path; - - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_C_COMPILER=clang" -D"CMAKE_CXX_COMPILER=clang++" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-arm64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-arm64" -D"PYTHON_MODULE_PATH=python" -D"USE_QT6=ON" -D"OPENSHOT_QT_API=pyqt6" -G Ninja -D"CMAKE_BUILD_TYPE:STRING=Release" + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_C_COMPILER=clang" -D"CMAKE_CXX_COMPILER=clang++" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-arm64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-arm64" -D"PYTHON_MODULE_PATH=python" -D"USE_QT6=ON" -D"OPENSHOT_QT_API=pyqt6" -G Ninja -D"CMAKE_BUILD_TYPE:STRING=Release" - cmake --build build --parallel $([Environment]::ProcessorCount) - ctest --test-dir build --output-on-failure -VV - cmake --install build @@ -190,10 +192,13 @@ windows-builder-x86: - $env:OpenCV_DIR = "$env:OPENCV_ROOT\lib\cmake\opencv4" - $env:Path = "$env:OPENCV_ROOT\bin;C:\msys64\mingw32\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; - $env:MSYSTEM = "MINGW32" + - $env:FFMPEGDIR = "C:/msys64/mingw32" + - if ((& "$env:FFMPEGDIR/bin/ffmpeg.exe" -version | Select-Object -First 1) -notmatch "^ffmpeg version 7[.]1[.]") { throw "FFmpeg 7.1 is required" } - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - - cmake -B build -S . -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32" + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" -DENABLE_GPU_FRAME=ON - cmake --build build --parallel $([Environment]::ProcessorCount) + - ctest --test-dir build --output-on-failure -VV - cmake --install build - $PROJECT_VERSION = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_VERSION_FULL "(.*)\"' | %{$_.Matches.Groups[1].value}) - $PROJECT_SO = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_SO_VERSION (.*)\)' | %{$_.Matches.Groups[1].value}) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ce956b844..fe836b8e5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -98,8 +98,11 @@ set(OPENSHOT_TESTS WaveEffect ) -if(ENABLE_GPU_FRAME AND CMAKE_SYSTEM_NAME STREQUAL "Linux") - list(APPEND OPENSHOT_TESTS VulkanPresentation VulkanCompositor) +if(ENABLE_GPU_FRAME) + list(APPEND OPENSHOT_TESTS VulkanCompositor) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND OPENSHOT_TESTS VulkanPresentation) + endif() endif() # The SIGPIPE crash-handler test is POSIX-only. Do not create an empty diff --git a/tests/FFmpegReader.cpp b/tests/FFmpegReader.cpp index a27a5810e..c02bc49cf 100644 --- a/tests/FFmpegReader.cpp +++ b/tests/FFmpegReader.cpp @@ -17,6 +17,11 @@ #include #include #include +// Windows CRT equivalents for these tests (all calls overwrite). +#ifdef _WIN32 +#define setenv(name, value, overwrite) _putenv_s(name, value) +#define unsetenv(name) _putenv_s(name, "") +#endif #include #include #include From 35efd38a3db323d48e369257ebc3dbe1623a802b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 30 Oct 2026 12:26:52 -0500 Subject: [PATCH 17/50] Fix Windows dependency paths and stabilize CI autogen --- .gitlab-ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4249b0e5..6fa40236b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -118,14 +118,17 @@ windows-builder-x64: - "if ($CI_COMMIT_REF_NAME -eq \"arm64-support\" -and -not (Select-String -Path \"build/install-x64/share/libopenshot-audio.env\" -Pattern \"^CI_COMMIT_REF_NAME:arm64-support$\" -Quiet)) { throw \"Matching arm64-support audio artifacts are required\" }" - $env:OPENCV_ROOT = "C:\msys64\mingw64\opencv-4.13.0" - $env:OpenCV_DIR = "$env:OPENCV_ROOT\lib\cmake\opencv4" - - $env:Path = "$env:OPENCV_ROOT\x64\mingw\bin;$env:OPENCV_ROOT\bin;C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; + - $env:Path = "$CI_PROJECT_DIR\build\install-x64\bin;$CI_PROJECT_DIR\build\src;$env:OPENCV_ROOT\x64\mingw\bin;$env:OPENCV_ROOT\bin;C:\msys64\mingw64\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; - $env:MSYSTEM = "MINGW64" - $env:FFMPEGDIR = "C:/msys64/mingw64" - if ((& "$env:FFMPEGDIR/bin/ffmpeg.exe" -version | Select-Object -First 1) -notmatch "^ffmpeg version 7[.]1[.]") { throw "FFmpeg 7.1 is required" } - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -DENABLE_GPU_FRAME=ON - - cmake --build build --parallel $([Environment]::ProcessorCount) + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -DENABLE_GPU_FRAME=ON + - cmake --build build --parallel 4 + - $testList = ctest --test-dir build --show-only=json-v1 | ConvertFrom-Json + - if ($LASTEXITCODE -ne 0 -or @($testList.tests).Count -lt 400) { throw "Incomplete unit test discovery; check runtime DLL paths" } + - Write-Host "Discovered $(@($testList.tests).Count) unit tests" - ctest --test-dir build --output-on-failure -VV - cmake --install build - $PROJECT_VERSION = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_VERSION_FULL "(.*)\"' | %{$_.Matches.Groups[1].value}) @@ -190,14 +193,17 @@ windows-builder-x86: - "if ($CI_COMMIT_REF_NAME -eq \"arm64-support\" -and -not (Select-String -Path \"build/install-x86/share/libopenshot-audio.env\" -Pattern \"^CI_COMMIT_REF_NAME:arm64-support$\" -Quiet)) { throw \"Matching arm64-support audio artifacts are required\" }" - $env:OPENCV_ROOT = "C:\msys64\mingw32\opencv-4.13.0" - $env:OpenCV_DIR = "$env:OPENCV_ROOT\lib\cmake\opencv4" - - $env:Path = "$env:OPENCV_ROOT\bin;C:\msys64\mingw32\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; + - $env:Path = "$CI_PROJECT_DIR\build\install-x86\bin;$CI_PROJECT_DIR\build\src;$env:OPENCV_ROOT\x64\mingw\bin;$env:OPENCV_ROOT\bin;C:\msys64\mingw32\bin;C:\msys64\usr\bin;C:\msys64\usr\local\bin;" + $env:Path; - $env:MSYSTEM = "MINGW32" - $env:FFMPEGDIR = "C:/msys64/mingw32" - if ((& "$env:FFMPEGDIR/bin/ffmpeg.exe" -version | Select-Object -First 1) -notmatch "^ffmpeg version 7[.]1[.]") { throw "FFmpeg 7.1 is required" } - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" -DENABLE_GPU_FRAME=ON - - cmake --build build --parallel $([Environment]::ProcessorCount) + - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" -DENABLE_GPU_FRAME=ON + - cmake --build build --parallel 4 + - $testList = ctest --test-dir build --show-only=json-v1 | ConvertFrom-Json + - if ($LASTEXITCODE -ne 0 -or @($testList.tests).Count -lt 400) { throw "Incomplete unit test discovery; check runtime DLL paths" } + - Write-Host "Discovered $(@($testList.tests).Count) unit tests" - ctest --test-dir build --output-on-failure -VV - cmake --install build - $PROJECT_VERSION = (Select-String -Path "CMakeLists.txt" -Pattern '^set\(PROJECT_VERSION_FULL "(.*)\"' | %{$_.Matches.Groups[1].value}) From cc21c06a98b57b8d55b03cc26110f0d06e6356fc Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 30 Oct 2026 12:27:29 -0500 Subject: [PATCH 18/50] Print GitLab server version in pipeline setup --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fa40236b..a3d776ccf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,7 @@ init-builders: tags: - gitlab script: + - echo "GitLab server version $CI_SERVER_VERSION" - send-ci-sqs linux - send-ci-sqs windows From 34068100240c63e139dc8b119104a285241a5909 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 30 Oct 2026 14:10:41 -0500 Subject: [PATCH 19/50] Share Mac video surfaces and enable Windows GPU preview --- .gitlab-ci.yml | 3 +- bindings/python/openshot.i | 7 +- src/CMakeLists.txt | 7 +- src/FFmpegReader.cpp | 55 +++++- src/FFmpegWriter.cpp | 46 +++++ src/GpuDevices.cpp | 26 ++- src/MacVulkanInterop.h | 23 +++ src/MacVulkanInterop.mm | 294 +++++++++++++++++++++++++++++ src/Qt/ExternalOpenGL.h | 116 ++++++++++++ src/Qt/VulkanOpenGLPresenter.cpp | 66 ++++++- src/Qt/VulkanOpenGLPresenterMac.mm | 172 +++++++++++++++++ src/VulkanCompositor.cpp | 32 +++- src/VulkanFrameImage.cpp | 20 +- tests/CMakeLists.txt | 5 +- tests/VulkanCompositor.cpp | 73 +++++++ tests/VulkanPresentation.cpp | 19 +- 16 files changed, 933 insertions(+), 31 deletions(-) create mode 100644 src/MacVulkanInterop.h create mode 100644 src/MacVulkanInterop.mm create mode 100644 src/Qt/ExternalOpenGL.h create mode 100644 src/Qt/VulkanOpenGLPresenterMac.mm diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3d776ccf..5d183274e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,7 +91,8 @@ mac-builder: - export PKG_CONFIG_PATH="$FFMPEGDIR/lib/pkgconfig:$PKG_CONFIG_PATH" - export DYLD_LIBRARY_PATH=$FFMPEGDIR/lib:$OPENCV_ROOT/lib:$DYLD_LIBRARY_PATH - mkdir -p build; cd build; - - cmake -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.12" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ + - test -f "$FFMPEGDIR/lib/libMoltenVK.dylib" && test -x "$FFMPEGDIR/bin/glslangValidator" + - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$FFMPEGDIR/include" -D"Vulkan_LIBRARY:FILEPATH=$FFMPEGDIR/lib/libMoltenVK.dylib" -D"GLSLANG_VALIDATOR:FILEPATH=$FFMPEGDIR/bin/glslangValidator" -DCMAKE_EXE_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_SHARED_LINKER_FLAGS="-stdlib=libc++" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_OSX_SYSROOT=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.0.sdk" -D"CMAKE_OSX_DEPLOYMENT_TARGET=10.15" -DCMAKE_PREFIX_PATH=/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/ -D"CMAKE_INSTALL_RPATH_USE_LINK_PATH=1" -D"ENABLE_RUBY=0" ../ - cmake --build . --parallel $(sysctl -n hw.ncpu) - make install - find "$CI_PROJECT_DIR/build/install-x64" -type f \( -name "*.dylib" -or -name "*.so" \) -exec sh -c 'for f do if ! otool -l "$f" | grep -q "$OPENCV_ROOT/lib"; then install_name_tool -add_rpath "$OPENCV_ROOT/lib" "$f" || true; fi; done' sh {} + diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index c65e5a04b..1fb38805c 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -110,9 +110,10 @@ class QWidget; #include "QtTextReader.h" #include "KeyFrame.h" #include "RendererBase.h" -#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) +#if defined(OPENSHOT_ENABLE_GPU_FRAME) #include "Qt/VulkanOpenGLPresenter.h" #include "VulkanCompositor.h" +#include "VulkanFrameImage.h" #include "GpuDevices.h" #endif #include "Settings.h" @@ -525,9 +526,9 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "KeyFrame.h" %include "AnimatedCurve.h" %include "RendererBase.h" -#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(LINUX) +#if defined(OPENSHOT_ENABLE_GPU_FRAME) %include "Qt/VulkanOpenGLPresenter.h" -namespace openshot { void ReleaseVulkanMemory(); } +namespace openshot { void ReleaseVulkanMemory(); uint64_t VulkanFrameReadbackCount(); } %include "GpuDevices.h" %template(GpuDeviceVector) std::vector; %template(StringVector) std::vector; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index eec506ee9..642f2bca3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -450,7 +450,12 @@ if(ENABLE_GPU_FRAME) target_sources(openshot PRIVATE VulkanFrameImage.cpp) target_sources(openshot PRIVATE VulkanCompositor.cpp) target_sources(openshot PRIVATE GpuDevices.cpp) - if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + if(APPLE) + enable_language(OBJCXX) + target_sources(openshot PRIVATE MacVulkanInterop.mm MacVulkanInterop.h Qt/VulkanOpenGLPresenterMac.mm Qt/VulkanOpenGLPresenter.h) + set_property(TARGET openshot PROPERTY OBJCXX_STANDARD 17) + target_link_libraries(openshot PRIVATE "-framework Foundation" "-framework CoreFoundation" objc "-framework Metal" "-framework CoreVideo" "-framework IOSurface" "-framework OpenGL") + else() target_sources(openshot PRIVATE Qt/VulkanOpenGLPresenter.cpp Qt/VulkanOpenGLPresenter.h) endif() find_package(Vulkan REQUIRED) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 992bbb7e1..2a5c65941 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -146,6 +146,13 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64 // Get hardware pix format static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts) { +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) + if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX) { + for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format) + if (*format == AV_PIX_FMT_VIDEOTOOLBOX) return *format; + return avcodec_default_get_format(ctx, pix_fmts); + } +#endif #ifdef OPENSHOT_ENABLE_GPU_FRAME if (ctx->hw_device_ctx && reinterpret_cast(ctx->hw_device_ctx->data)->type == AV_HWDEVICE_TYPE_VULKAN) { for (auto format = pix_fmts; *format != AV_PIX_FMT_NONE; ++format) @@ -331,7 +338,12 @@ void FFmpegReader::Open() { #ifdef OPENSHOT_ENABLE_GPU_FRAME hw_de_supported = 0; for (int index = 0; const AVCodecHWConfig* config = avcodec_get_hw_config(pCodec, index); ++index) { - if (config->device_type == AV_HWDEVICE_TYPE_VULKAN && + if (config->device_type == +#ifdef __APPLE__ + AV_HWDEVICE_TYPE_VIDEOTOOLBOX && +#else + AV_HWDEVICE_TYPE_VULKAN && +#endif (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) { hw_de_supported = 1; break; @@ -444,12 +456,16 @@ void FFmpegReader::Open() { } #ifdef OPENSHOT_ENABLE_GPU_FRAME +#ifdef __APPLE__ + hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; +#else hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN; +#endif adapter_ptr = nullptr; #endif hw_device_ctx = NULL; // Here the first hardware initialisations are made -#ifdef OPENSHOT_ENABLE_GPU_FRAME +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && !defined(__APPLE__) if (CreateSharedVulkanDevice(&hw_device_ctx) >= 0) { #else if (av_hwdevice_ctx_create(&hw_device_ctx, hw_de_av_device_type, adapter_ptr, NULL, 0) >= 0) { @@ -1636,7 +1652,10 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL // Get the format from the variables set in get_hw_dec_format -#ifdef OPENSHOT_ENABLE_GPU_FRAME +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) + hw_de_av_pix_fmt = AV_PIX_FMT_VIDEOTOOLBOX; + hw_de_av_device_type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX; +#elif defined(OPENSHOT_ENABLE_GPU_FRAME) hw_de_av_pix_fmt = AV_PIX_FMT_VULKAN; hw_de_av_device_type = AV_HWDEVICE_TYPE_VULKAN; #else @@ -1711,7 +1730,11 @@ bool FFmpegReader::GetAVFrame() { if (hw_de_on && hw_de_supported) { int err; #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (next_frame2->format == AV_PIX_FMT_VULKAN) { + if ((next_frame2->format == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX +#endif + )) { decoded_frame = next_frame2; } else #endif @@ -1753,7 +1776,11 @@ bool FFmpegReader::GetAVFrame() { decoded_frame = next_frame2; } - if (!decoded_frame->data[0]) { + if (!decoded_frame->data[0] +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) + && !(decoded_frame->format == AV_PIX_FMT_VIDEOTOOLBOX && decoded_frame->data[3]) +#endif + ) { Logger::Instance()->AppendDebugMethod( "FFmpegReader::GetAVFrame (Decoded frame missing image data)", "format", decoded_frame->format, @@ -1770,7 +1797,11 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL if (hw_de_on && hw_de_supported && !force_sw_decode) { #ifdef OPENSHOT_ENABLE_GPU_FRAME - hw_decode_succeeded = hw_decode_succeeded || next_frame2->format == AV_PIX_FMT_VULKAN; + hw_decode_succeeded = hw_decode_succeeded || (next_frame2->format == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX +#endif + ); #else hw_decode_succeeded = true; #endif @@ -1785,7 +1816,11 @@ bool FFmpegReader::GetAVFrame() { const int decoded_width = decoded_frame->width > 0 ? decoded_frame->width : info.width; const int decoded_height = decoded_frame->height > 0 ? decoded_frame->height : info.height; #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (decoded_pix_fmt == AV_PIX_FMT_VULKAN) { + if (decoded_pix_fmt == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || decoded_pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX +#endif + ) { if (av_frame_ref(pFrame, decoded_frame) < 0) throw OutOfMemory("Failed to retain Vulkan frame", path); } else { @@ -2113,7 +2148,11 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (pFrame && pFrame->format == AV_PIX_FMT_VULKAN) { + if (pFrame && (pFrame->format == AV_PIX_FMT_VULKAN +#ifdef __APPLE__ + || pFrame->format == AV_PIX_FMT_VIDEOTOOLBOX +#endif + )) { try { if (!vulkan_processor) vulkan_processor = std::make_shared(); auto f = CreateFrame(current_frame); diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index cf7d31592..e81eeb79a 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -34,6 +34,9 @@ #include "VulkanFrameImage.h" #include "VulkanCompositor.h" #include "VulkanEncodeConverter.h" +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif #endif #include "OpenMPUtilities.h" #include "Settings.h" @@ -2702,6 +2705,49 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { if (src_w == 1 && src_h == 1) return; +#if defined(OPENSHOT_ENABLE_GPU_FRAME) && defined(__APPLE__) && USE_HW_ACCEL + if (!gpu_export_failed && hw_en_on && hw_en_supported && + video_codec_ctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX && video_codec_ctx->hw_frames_ctx) { + auto surface = frame->GetDeferredImage(); + if (surface) { + AVFrame* encoded = av_frame_alloc(); + try { + if (!encoded) throw std::bad_alloc(); + auto native = std::dynamic_pointer_cast(surface); + const auto* context = native ? reinterpret_cast(native->GetFrame()->hw_frames_ctx->data) : nullptr; + if (src_w != info.width || src_h != info.height || !context || context->sw_format != AV_PIX_FMT_RGBA) { + QTransform scale; + scale.scale(double(info.width) / src_w, double(info.height) / src_h); + surface = RenderVulkanImage(surface, nullptr, info.width, info.height, scale, 1.0, Qt::black); + } + // Render directly into the encoder's own Core Video pixel buffer. + // No CPU download, upload, or separate NV12 image copy is needed. + const int allocated = av_hwframe_get_buffer(video_codec_ctx->hw_frames_ctx, encoded, 0); + if (allocated < 0) throw std::runtime_error("Allocate VideoToolbox encoder frame: " + av_err2string(allocated)); + std::unique_ptr target(ImportVideoToolboxFrame(encoded, true), + [](AVFrame* value) { av_frame_free(&value); }); + auto converted = std::dynamic_pointer_cast(ConvertVulkanToNV12(surface, target.get())); + if (!converted) throw std::runtime_error("No shared VideoToolbox conversion result"); + // VideoToolbox does not consume Vulkan timeline semaphores. Retire + // GPU writes before handing the same IOSurface to its encoder. + WaitMacVulkanFrame(converted->GetFrame()); + const int copied = av_frame_copy_props(encoded, converted->GetFrame()); + if (copied < 0) throw std::bad_alloc(); + add_avframe(frame, encoded); + encoded = nullptr; // add_avframe owns the retained encoder frame. + if (!gpu_export_logged) { + gpu_export_logged = true; + Logger::Instance()->Log("Vulkan export: encoder-owned VideoToolbox IOSurface (no CPU copy)", Logger::LevelInfo); + } + return; + } catch (const std::exception& error) { + av_frame_free(&encoded); + gpu_export_failed = true; + Logger::Instance()->Log(std::string("VideoToolbox GPU export CPU fallback: ") + error.what(), Logger::LevelWarning); + } + } + } +#endif #if defined(OPENSHOT_ENABLE_GPU_FRAME) && USE_HW_ACCEL // Native Vulkan writes directly into encoder-owned NV12. NVENC retains its // device-to-device transfer path; other encoders use the established CPU path. diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index e6969b908..33201638b 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -148,13 +148,13 @@ bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format if (!codec) return false; AVCodecContext* context = avcodec_alloc_context3(codec); if (!context) return false; - context->width = 256; - context->height = 256; + context->width = 640; + context->height = 360; context->time_base = {1, 30}; context->framerate = {30, 1}; context->bit_rate = 1000000; bool opened = false; - if (hw_format == AV_PIX_FMT_VAAPI || hw_format == AV_PIX_FMT_VULKAN) { + if (hw_format == AV_PIX_FMT_VAAPI || hw_format == AV_PIX_FMT_VULKAN || hw_format == AV_PIX_FMT_VIDEOTOOLBOX) { AVBufferRef* frames = av_hwframe_ctx_alloc(device); if (frames) { auto* fc = reinterpret_cast(frames->data); @@ -196,10 +196,20 @@ int CreateSharedVulkanDevice(AVBufferRef** output) { if (shared.tried) return AVERROR(ENODEV); shared.tried = true; +#ifdef __APPLE__ + // Work around imported-texture argument-buffer writes on older Intel + // Metal drivers. An explicit user setting still takes precedence. + setenv("MVK_CONFIG_USE_METAL_ARGUMENT_BUFFERS", "0", 0); +#endif int result = AVERROR(ENODEV); for (const auto& device : VulkanDeviceCandidates()) { + AVDictionary* options = nullptr; +#ifdef __APPLE__ + av_dict_set(&options, "device_extensions", "VK_EXT_metal_objects", 0); +#endif result = av_hwdevice_ctx_create(&shared.device, AV_HWDEVICE_TYPE_VULKAN, - device.empty() ? nullptr : device.c_str(), nullptr, 0); + device.empty() ? nullptr : device.c_str(), options, 0); + av_dict_free(&options); if (result >= 0) break; } @@ -269,6 +279,14 @@ std::vector GpuEncoders() { if (EncoderOpens(name, cuda, AV_PIX_FMT_CUDA)) encoders.push_back(name); av_buffer_unref(&cuda); } +#ifdef __APPLE__ + AVBufferRef* videotoolbox = nullptr; + if (av_hwdevice_ctx_create(&videotoolbox, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0) >= 0) { + for (const char* name : {"h264_videotoolbox", "hevc_videotoolbox"}) + if (EncoderOpens(name, videotoolbox, AV_PIX_FMT_VIDEOTOOLBOX)) encoders.push_back(name); + av_buffer_unref(&videotoolbox); + } +#endif AVBufferRef* vaapi = nullptr; if (!gpu.render_node.empty() && av_hwdevice_ctx_create(&vaapi, AV_HWDEVICE_TYPE_VAAPI, gpu.render_node.c_str(), nullptr, 0) >= 0) { diff --git a/src/MacVulkanInterop.h b/src/MacVulkanInterop.h new file mode 100644 index 000000000..a1a4d53f4 --- /dev/null +++ b/src/MacVulkanInterop.h @@ -0,0 +1,23 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_MAC_VULKAN_INTEROP_H +#define OPENSHOT_MAC_VULKAN_INTEROP_H +#include +struct AVFrame; +struct AVBufferRef; +namespace openshot { +// Internal Apple bridge. Returned AVFrames are owned by the caller. Hardware +// surfaces retain their CVPixelBuffers until all Vulkan submissions have retired. +AVFrame* ImportVideoToolboxFrame(const AVFrame* frame, bool writable = false); +AVFrame* AllocateMacVulkanFrame(AVBufferRef* device, int width, int height, bool nv12); +uint64_t MacVulkanDeviceRegistryID(const AVFrame* frame); +int64_t MacVulkanFrameBytes(const AVFrame* frame); +bool IsMacVulkanFrame(const AVFrame* frame); +void WaitMacVulkanFrame(const AVFrame* frame); +// Borrowed CVPixelBufferRef; valid while the AVFrame is retained. +void* MacVulkanPixelBuffer(const AVFrame* frame); +// Explicit CPU fallback only. Returns an FFmpeg status; never used by GPU export. +int ReadMacVulkanFrame(AVFrame* destination, const AVFrame* source); +void ReleaseMacVulkanPools(); +} +#endif diff --git a/src/MacVulkanInterop.mm b/src/MacVulkanInterop.mm new file mode 100644 index 000000000..5f119e167 --- /dev/null +++ b/src/MacVulkanInterop.mm @@ -0,0 +1,294 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#define VK_USE_PLATFORM_METAL_EXT +#import +#import +#import +#include "MacVulkanInterop.h" +#include "GpuDevices.h" +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +} +namespace openshot { +namespace { +const char native_pool_tag = 0; +void AvCheck(int r, const char* what) { if (r < 0) throw std::runtime_error(what); } +void VkCheck(VkResult r, const char* what) { if (r != VK_SUCCESS) throw std::runtime_error(what); } +struct Pools { + std::mutex mutex; + std::map, AVBufferRef*> frames; + ~Pools() { for (auto& pair : frames) av_buffer_unref(&pair.second); } +}; +Pools& NativePools() { static Pools pools; return pools; } +AVBufferRef* Pool(AVBufferRef* device, int width, int height, bool nv12) { + auto& pools = NativePools(); + std::lock_guard lock(pools.mutex); + auto key = std::make_tuple(reinterpret_cast(device->data), width, height, nv12); + auto& ref = pools.frames[key]; + if (!ref) { + ref = av_hwframe_ctx_alloc(device); + if (!ref) throw std::bad_alloc(); + auto* fc = reinterpret_cast(ref->data); + fc->format = AV_PIX_FMT_VULKAN; + fc->sw_format = nv12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_RGBA; + fc->width = width; fc->height = height; + fc->user_opaque = const_cast(&native_pool_tag); + auto* vc = static_cast(fc->hwctx); + vc->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; + vc->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + const int result = av_hwframe_ctx_init(ref); + if (result < 0) { av_buffer_unref(&ref); pools.frames.erase(key); AvCheck(result, "Initialize Mac Vulkan frame pool"); } + } + AVBufferRef* result = av_buffer_ref(ref); + if (!result) throw std::bad_alloc(); + return result; +} +struct NativeFrame { + AVFrame* allocation = nullptr; + AVVkFrame original{}; + AVVkFrame* image = nullptr; + AVHWFramesContext* fc = nullptr; + AVVulkanFramesContext* vc = nullptr; + AVVulkanDeviceContext* vk = nullptr; + CVPixelBufferRef pixel = nullptr; + CVMetalTextureCacheRef cache = nullptr; + std::array textures{}; + std::array, 2> direct_textures{}; + id Texture(int plane) const { + return direct_textures[plane] ?: CVMetalTextureGetTexture(textures[plane]); + } + std::array imported{}; + uint64_t registry_id = 0; + int count = 0; + bool substituted = false; + void Wait() const { + vc->lock_frame(fc, image); + VkSemaphoreWaitInfo wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + wait.semaphoreCount = count; + wait.pSemaphores = image->sem; + wait.pValues = image->sem_value; + const auto result = vkWaitSemaphores(vk->act_dev, &wait, UINT64_MAX); + vc->unlock_frame(fc, image); + VkCheck(result, "Wait for Mac surface GPU access"); + } + ~NativeFrame() { + if (substituted) { + // Keep the normal FFmpeg allocation and its private synchronization + // internals intact. Its pool callback is the only supported way to + // release an AVVkFrame; no reliance on FFmpeg's private struct ABI. + try { Wait(); } catch (...) { /* Device loss: retire owned objects. */ } + } + if (vk) for (auto img : imported) if (img) vkDestroyImage(vk->act_dev, img, nullptr); + if (substituted) { + for (int i = 0; i < count; ++i) original.sem_value[i] = image->sem_value[i]; + *image = original; + } + av_frame_free(&allocation); + for (auto texture : textures) if (texture) CFRelease(texture); + for (auto texture : direct_textures) [texture release]; + if (cache) CFRelease(cache); + if (pixel) CVPixelBufferRelease(pixel); + } +}; +NativeFrame* State(const AVFrame* frame) { + if (!frame || frame->format != AV_PIX_FMT_VULKAN || !frame->hw_frames_ctx || !frame->buf[0]) return nullptr; + auto* fc = reinterpret_cast(frame->hw_frames_ctx->data); + if (fc->user_opaque != &native_pool_tag) return nullptr; + return static_cast(av_buffer_get_opaque(frame->buf[0])); +} +AVFrame* Wrap(AVBufferRef* device, CVPixelBufferRef pixel, bool nv12, bool writable) { + @autoreleasepool { + const int width = static_cast(CVPixelBufferGetWidth(pixel)); + const int height = static_cast(CVPixelBufferGetHeight(pixel)); + if (width <= 0 || height <= 0 || !CVPixelBufferGetIOSurface(pixel)) + throw std::invalid_argument("Mac GPU frames require IOSurface-backed pixels"); + auto state = std::make_unique(); + state->pixel = CVPixelBufferRetain(pixel); + state->count = nv12 ? 2 : 1; + state->allocation = av_frame_alloc(); + if (!state->allocation) throw std::bad_alloc(); + AVBufferRef* pool = Pool(device, width, height, nv12); + const int allocated = av_hwframe_get_buffer(pool, state->allocation, 0); + av_buffer_unref(&pool); + AvCheck(allocated, "Allocate Mac Vulkan frame synchronization"); + state->fc = reinterpret_cast(state->allocation->hw_frames_ctx->data); + state->vc = static_cast(state->fc->hwctx); + state->vk = static_cast(state->fc->device_ctx->hwctx); + state->image = reinterpret_cast(state->allocation->data[0]); + state->original = *state->image; + auto exportObjects = reinterpret_cast(vkGetDeviceProcAddr(state->vk->act_dev, "vkExportMetalObjectsEXT")); + if (!exportObjects) throw std::runtime_error("Vulkan device lacks VK_EXT_metal_objects"); + VkExportMetalDeviceInfoEXT metal{VK_STRUCTURE_TYPE_EXPORT_METAL_DEVICE_INFO_EXT}; + VkExportMetalObjectsInfoEXT objects{VK_STRUCTURE_TYPE_EXPORT_METAL_OBJECTS_INFO_EXT}; + objects.pNext = &metal; + exportObjects(state->vk->act_dev, &objects); + state->registry_id = metal.mtlDevice.registryID; + if (!metal.mtlDevice) throw std::runtime_error("No Metal device for Vulkan"); + if (CVMetalTextureCacheCreate(kCFAllocatorDefault, nullptr, metal.mtlDevice, nullptr, &state->cache) != kCVReturnSuccess) + throw std::runtime_error("Create Core Video Metal texture cache"); + NSDictionary* attributes = @{(id)kCVMetalTextureUsage: @(MTLTextureUsageShaderRead | + (writable ? MTLTextureUsageShaderWrite : 0))}; + for (int i = 0; i < state->count; ++i) { + const auto w = nv12 ? CVPixelBufferGetWidthOfPlane(pixel, i) : size_t(width); + const auto h = nv12 ? CVPixelBufferGetHeightOfPlane(pixel, i) : size_t(height); + const auto format = nv12 ? (i ? MTLPixelFormatRG8Unorm : MTLPixelFormatR8Unorm) : MTLPixelFormatRGBA8Unorm; + if (nv12) { + if (CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, state->cache, pixel, + (CFDictionaryRef)attributes, format, w, h, i, &state->textures[i]) != kCVReturnSuccess) + throw std::runtime_error("Map Core Video pixel planes to Metal"); + } else { + // Core Video cannot allocate Metal-compatible 32RGBA on older + // Macs. Allocate a 32BGRA IOSurface and give Metal an RGBA view; + // the compositor and CPU fallback both retain RGBA byte order. + auto* descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format + width:w height:h mipmapped:NO]; + descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite; + state->direct_textures[i] = [metal.mtlDevice newTextureWithDescriptor:descriptor + iosurface:CVPixelBufferGetIOSurface(pixel) plane:i]; + if (!state->direct_textures[i]) throw std::runtime_error("Create RGBA IOSurface texture view"); + } + VkImportMetalTextureInfoEXT texture{VK_STRUCTURE_TYPE_IMPORT_METAL_TEXTURE_INFO_EXT}; + texture.plane = VK_IMAGE_ASPECT_COLOR_BIT; + texture.mtlTexture = state->Texture(i); + VkImageCreateInfo create{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + create.pNext = &texture; create.imageType = VK_IMAGE_TYPE_2D; + create.format = nv12 ? (i ? VK_FORMAT_R8G8_UNORM : VK_FORMAT_R8_UNORM) : VK_FORMAT_R8G8B8A8_UNORM; + create.extent = {uint32_t(w), uint32_t(h), 1}; create.mipLevels = create.arrayLayers = 1; + create.samples = VK_SAMPLE_COUNT_1_BIT; create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.usage = VK_IMAGE_USAGE_SAMPLED_BIT | (writable ? VK_IMAGE_USAGE_STORAGE_BIT : 0); + create.initialLayout = writable ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_PREINITIALIZED; + VkCheck(vkCreateImage(state->vk->act_dev, &create, nullptr, &state->imported[i]), "Import shared Mac texture into Vulkan"); + } + for (int i = 0; i < state->count; ++i) { + state->image->img[i] = state->imported[i]; + state->image->mem[i] = VK_NULL_HANDLE; + state->image->size[i] = nv12 ? CVPixelBufferGetBytesPerRowOfPlane(pixel, i) * CVPixelBufferGetHeightOfPlane(pixel, i) + : CVPixelBufferGetBytesPerRow(pixel) * height; + state->image->offset[i] = 0; + state->image->layout[i] = writable ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_PREINITIALIZED; + state->image->access[i] = static_cast(writable ? 0 : VK_ACCESS_MEMORY_WRITE_BIT); + state->image->queue_family[i] = VK_QUEUE_FAMILY_IGNORED; + } + state->substituted = true; + AVFrame* result = av_frame_alloc(); + if (!result) throw std::bad_alloc(); + result->format = AV_PIX_FMT_VULKAN; result->width = width; result->height = height; + result->hw_frames_ctx = av_buffer_ref(state->allocation->hw_frames_ctx); + result->data[0] = reinterpret_cast(state->image); + result->buf[0] = av_buffer_create(result->data[0], sizeof(AVVkFrame), + [](void* opaque, uint8_t*) { delete static_cast(opaque); }, state.get(), 0); + if (!result->hw_frames_ctx || !result->buf[0]) { + // Do not let the AVBuffer callback race the still-owning unique_ptr. + if (result->buf[0]) state.release(); + av_frame_free(&result); + throw std::bad_alloc(); + } + state.release(); + return result; + } +} +} +uint64_t MacVulkanDeviceRegistryID(const AVFrame* frame) { auto* state = State(frame); return state ? state->registry_id : 0; } +int64_t MacVulkanFrameBytes(const AVFrame* frame) { + auto* state = State(frame); + if (!state) return 0; + uint64_t size = CVPixelBufferGetDataSize(state->pixel); + // The FFmpeg synchronization allocation is retained alongside the imported + // surface. Include both, rather than letting GPU frames escape cache limits. + for (int i = 0; i < state->count; ++i) { + bool counted = false; + for (int j = 0; j < i; ++j) counted |= state->original.mem[j] == state->original.mem[i]; + if (!counted) size += state->original.size[i]; + } + return static_cast(std::min(size, std::numeric_limits::max())); +} +bool IsMacVulkanFrame(const AVFrame* frame) { return State(frame) != nullptr; } +void WaitMacVulkanFrame(const AVFrame* frame) { + auto* state = State(frame); + if (!state) throw std::invalid_argument("Not a shared Mac Vulkan frame"); + state->Wait(); +} +void* MacVulkanPixelBuffer(const AVFrame* frame) { auto* state = State(frame); return state ? state->pixel : nullptr; } +AVFrame* ImportVideoToolboxFrame(const AVFrame* frame, bool writable) { + if (!frame || frame->format != AV_PIX_FMT_VIDEOTOOLBOX || !frame->data[3]) + throw std::invalid_argument("Expected a VideoToolbox frame"); + auto pixel = reinterpret_cast(frame->data[3]); + const auto type = CVPixelBufferGetPixelFormatType(pixel); + if (type != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && type != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) + throw std::runtime_error("Mac Vulkan currently supports 8-bit NV12 decode/encode surfaces"); + AVBufferRef* device = nullptr; + AvCheck(CreateSharedVulkanDevice(&device), "Create Mac compositor device"); + AVFrame* result = nullptr; + try { result = Wrap(device, pixel, true, writable); } catch (...) { av_buffer_unref(&device); throw; } + av_buffer_unref(&device); + const int props = av_frame_copy_props(result, frame); + if (props < 0) { av_frame_free(&result); AvCheck(props, "Copy VideoToolbox metadata"); } + return result; +} +AVFrame* AllocateMacVulkanFrame(AVBufferRef* device, int width, int height, bool nv12) { + @autoreleasepool { + NSDictionary* attrs = @{(id)kCVPixelBufferMetalCompatibilityKey: @YES, (id)kCVPixelBufferIOSurfacePropertiesKey: @{}}; + CVPixelBufferRef pixel = nullptr; + const auto type = nv12 ? kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange : kCVPixelFormatType_32BGRA; + if (CVPixelBufferCreate(kCFAllocatorDefault, width, height, type, (CFDictionaryRef)attrs, &pixel) != kCVReturnSuccess) + throw std::runtime_error("Allocate shared compositor IOSurface"); + AVFrame* frame = nullptr; + try { frame = Wrap(device, pixel, nv12, true); } catch (...) { CVPixelBufferRelease(pixel); throw; } + CVPixelBufferRelease(pixel); + return frame; + } +} +int ReadMacVulkanFrame(AVFrame* destination, const AVFrame* source) { + auto* state = State(source); + if (!state) return AVERROR(EINVAL); + try { state->Wait(); } catch (...) { return AVERROR_EXTERNAL; } + destination->format = state->fc->sw_format; + destination->width = source->width; destination->height = source->height; + int result = av_frame_get_buffer(destination, 32); + if (result < 0) return result; + // Managed Metal textures need an explicit GPU-to-CPU synchronization for + // materialization. This is intentionally absent from preview and export. + @autoreleasepool { + auto first = state->Texture(0); + id queue = [first.device newCommandQueue]; + if (!queue) return AVERROR_EXTERNAL; + id command = [queue commandBuffer]; + id blit = [command blitCommandEncoder]; + for (int i = 0; i < state->count; ++i) { + auto texture = state->Texture(i); + if (texture.storageMode == MTLStorageModeManaged) [blit synchronizeResource:texture]; + } + [blit endEncoding]; [command commit]; [command waitUntilCompleted]; + const bool completed = command.status == MTLCommandBufferStatusCompleted; + [queue release]; + if (!completed) return AVERROR_EXTERNAL; + } + if (CVPixelBufferLockBaseAddress(state->pixel, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return AVERROR_EXTERNAL; + const bool nv12 = destination->format == AV_PIX_FMT_NV12; + const uint8_t* data[4]{}; int strides[4]{}; + for (int i = 0; i < (nv12 ? 2 : 1); ++i) { + data[i] = static_cast(nv12 ? CVPixelBufferGetBaseAddressOfPlane(state->pixel, i) : CVPixelBufferGetBaseAddress(state->pixel)); + strides[i] = static_cast(nv12 ? CVPixelBufferGetBytesPerRowOfPlane(state->pixel, i) : CVPixelBufferGetBytesPerRow(state->pixel)); + } + av_image_copy(destination->data, destination->linesize, data, strides, AVPixelFormat(destination->format), source->width, source->height); + CVPixelBufferUnlockBaseAddress(state->pixel, kCVPixelBufferLock_ReadOnly); + return 0; +} +void ReleaseMacVulkanPools() { + auto& pools = NativePools(); std::lock_guard lock(pools.mutex); + for (auto& pair : pools.frames) av_buffer_unref(&pair.second); + pools.frames.clear(); +} +} diff --git a/src/Qt/ExternalOpenGL.h b/src/Qt/ExternalOpenGL.h new file mode 100644 index 000000000..928190c83 --- /dev/null +++ b/src/Qt/ExternalOpenGL.h @@ -0,0 +1,116 @@ +/* +** Copyright (c) 2013-2018 The Khronos Group Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and/or associated documentation files (the +** "Materials"), to deal in the Materials without restriction, including +** without limitation the rights to use, copy, modify, merge, publish, +** distribute, sublicense, and/or sell copies of the Materials, and to +** permit persons to whom the Materials are furnished to do so, subject to +** the following conditions: +** +** The above copyright notice and this permission notice shall be included +** in all copies or substantial portions of the Materials. +** +** THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +** EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +** MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +** IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +** CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +** TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +** MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. +*/ +// Subset of Khronos OpenGL-Registry api/GL/glext.h for Qt 5 SDKs whose +// extension headers predate external memory and semaphore interop. +#ifndef OPENSHOT_EXTERNAL_OPENGL_H +#define OPENSHOT_EXTERNAL_OPENGL_H +#include + +#ifndef GL_EXT_memory_object +#define GL_EXT_memory_object 1 +#define GL_TEXTURE_TILING_EXT 0x9580 +#define GL_DEDICATED_MEMORY_OBJECT_EXT 0x9581 +#define GL_PROTECTED_MEMORY_OBJECT_EXT 0x959B +#define GL_NUM_TILING_TYPES_EXT 0x9582 +#define GL_TILING_TYPES_EXT 0x9583 +#define GL_OPTIMAL_TILING_EXT 0x9584 +#define GL_LINEAR_TILING_EXT 0x9585 +#define GL_NUM_DEVICE_UUIDS_EXT 0x9596 +#define GL_DEVICE_UUID_EXT 0x9597 +#define GL_DRIVER_UUID_EXT 0x9598 +#define GL_UUID_SIZE_EXT 16 +typedef void (APIENTRYP PFNGLGETUNSIGNEDBYTEVEXTPROC) (GLenum pname, GLubyte *data); +typedef void (APIENTRYP PFNGLGETUNSIGNEDBYTEI_VEXTPROC) (GLenum target, GLuint index, GLubyte *data); +typedef void (APIENTRYP PFNGLDELETEMEMORYOBJECTSEXTPROC) (GLsizei n, const GLuint *memoryObjects); +typedef GLboolean (APIENTRYP PFNGLISMEMORYOBJECTEXTPROC) (GLuint memoryObject); +typedef void (APIENTRYP PFNGLCREATEMEMORYOBJECTSEXTPROC) (GLsizei n, GLuint *memoryObjects); +typedef void (APIENTRYP PFNGLMEMORYOBJECTPARAMETERIVEXTPROC) (GLuint memoryObject, GLenum pname, const GLint *params); +typedef void (APIENTRYP PFNGLGETMEMORYOBJECTPARAMETERIVEXTPROC) (GLuint memoryObject, GLenum pname, GLint *params); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM2DEXTPROC) (GLenum target, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM2DMULTISAMPLEEXTPROC) (GLenum target, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM3DEXTPROC) (GLenum target, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM3DMULTISAMPLEEXTPROC) (GLenum target, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLBUFFERSTORAGEMEMEXTPROC) (GLenum target, GLsizeiptr size, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM2DEXTPROC) (GLuint texture, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM2DMULTISAMPLEEXTPROC) (GLuint texture, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM3DEXTPROC) (GLuint texture, GLsizei levels, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM3DMULTISAMPLEEXTPROC) (GLuint texture, GLsizei samples, GLenum internalFormat, GLsizei width, GLsizei height, GLsizei depth, GLboolean fixedSampleLocations, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLNAMEDBUFFERSTORAGEMEMEXTPROC) (GLuint buffer, GLsizeiptr size, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXSTORAGEMEM1DEXTPROC) (GLenum target, GLsizei levels, GLenum internalFormat, GLsizei width, GLuint memory, GLuint64 offset); +typedef void (APIENTRYP PFNGLTEXTURESTORAGEMEM1DEXTPROC) (GLuint texture, GLsizei levels, GLenum internalFormat, GLsizei width, GLuint memory, GLuint64 offset); +#endif /* GL_EXT_memory_object */ + +#ifndef GL_EXT_memory_object_fd +#define GL_EXT_memory_object_fd 1 +#define GL_HANDLE_TYPE_OPAQUE_FD_EXT 0x9586 +typedef void (APIENTRYP PFNGLIMPORTMEMORYFDEXTPROC) (GLuint memory, GLuint64 size, GLenum handleType, GLint fd); +#endif /* GL_EXT_memory_object_fd */ + +#ifndef GL_EXT_memory_object_win32 +#define GL_EXT_memory_object_win32 1 +#define GL_HANDLE_TYPE_OPAQUE_WIN32_EXT 0x9587 +#define GL_HANDLE_TYPE_OPAQUE_WIN32_KMT_EXT 0x9588 +#define GL_DEVICE_LUID_EXT 0x9599 +#define GL_DEVICE_NODE_MASK_EXT 0x959A +#define GL_LUID_SIZE_EXT 8 +#define GL_HANDLE_TYPE_D3D12_TILEPOOL_EXT 0x9589 +#define GL_HANDLE_TYPE_D3D12_RESOURCE_EXT 0x958A +#define GL_HANDLE_TYPE_D3D11_IMAGE_EXT 0x958B +#define GL_HANDLE_TYPE_D3D11_IMAGE_KMT_EXT 0x958C +typedef void (APIENTRYP PFNGLIMPORTMEMORYWIN32HANDLEEXTPROC) (GLuint memory, GLuint64 size, GLenum handleType, void *handle); +typedef void (APIENTRYP PFNGLIMPORTMEMORYWIN32NAMEEXTPROC) (GLuint memory, GLuint64 size, GLenum handleType, const void *name); +#endif /* GL_EXT_memory_object_win32 */ + +#ifndef GL_EXT_semaphore +#define GL_EXT_semaphore 1 +#define GL_LAYOUT_GENERAL_EXT 0x958D +#define GL_LAYOUT_COLOR_ATTACHMENT_EXT 0x958E +#define GL_LAYOUT_DEPTH_STENCIL_ATTACHMENT_EXT 0x958F +#define GL_LAYOUT_DEPTH_STENCIL_READ_ONLY_EXT 0x9590 +#define GL_LAYOUT_SHADER_READ_ONLY_EXT 0x9591 +#define GL_LAYOUT_TRANSFER_SRC_EXT 0x9592 +#define GL_LAYOUT_TRANSFER_DST_EXT 0x9593 +#define GL_LAYOUT_DEPTH_READ_ONLY_STENCIL_ATTACHMENT_EXT 0x9530 +#define GL_LAYOUT_DEPTH_ATTACHMENT_STENCIL_READ_ONLY_EXT 0x9531 +typedef void (APIENTRYP PFNGLGENSEMAPHORESEXTPROC) (GLsizei n, GLuint *semaphores); +typedef void (APIENTRYP PFNGLDELETESEMAPHORESEXTPROC) (GLsizei n, const GLuint *semaphores); +typedef GLboolean (APIENTRYP PFNGLISSEMAPHOREEXTPROC) (GLuint semaphore); +typedef void (APIENTRYP PFNGLSEMAPHOREPARAMETERUI64VEXTPROC) (GLuint semaphore, GLenum pname, const GLuint64 *params); +typedef void (APIENTRYP PFNGLGETSEMAPHOREPARAMETERUI64VEXTPROC) (GLuint semaphore, GLenum pname, GLuint64 *params); +typedef void (APIENTRYP PFNGLWAITSEMAPHOREEXTPROC) (GLuint semaphore, GLuint numBufferBarriers, const GLuint *buffers, GLuint numTextureBarriers, const GLuint *textures, const GLenum *srcLayouts); +typedef void (APIENTRYP PFNGLSIGNALSEMAPHOREEXTPROC) (GLuint semaphore, GLuint numBufferBarriers, const GLuint *buffers, GLuint numTextureBarriers, const GLuint *textures, const GLenum *dstLayouts); +#endif /* GL_EXT_semaphore */ + +#ifndef GL_EXT_semaphore_fd +#define GL_EXT_semaphore_fd 1 +typedef void (APIENTRYP PFNGLIMPORTSEMAPHOREFDEXTPROC) (GLuint semaphore, GLenum handleType, GLint fd); +#endif /* GL_EXT_semaphore_fd */ + +#ifndef GL_EXT_semaphore_win32 +#define GL_EXT_semaphore_win32 1 +#define GL_HANDLE_TYPE_D3D12_FENCE_EXT 0x9594 +#define GL_D3D12_FENCE_VALUE_EXT 0x9595 +typedef void (APIENTRYP PFNGLIMPORTSEMAPHOREWIN32HANDLEEXTPROC) (GLuint semaphore, GLenum handleType, void *handle); +typedef void (APIENTRYP PFNGLIMPORTSEMAPHOREWIN32NAMEEXTPROC) (GLuint semaphore, GLenum handleType, const void *name); +#endif /* GL_EXT_semaphore_win32 */ +#endif diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 8a308b8c9..3e38f1377 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -13,8 +13,16 @@ #include #include #include -#include +#include "ExternalOpenGL.h" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#define VK_USE_PLATFORM_WIN32_KHR +#else #include +#endif #include #include #include @@ -99,7 +107,11 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { // Multi-plane decode surfaces must first go through the RGBA compositor. if (fc->sw_format != AV_PIX_FMT_RGBA || vfc->format[0] != VK_FORMAT_R8G8B8A8_UNORM || image->img[1]) return false; +#ifdef _WIN32 + if (!context->hasExtension("GL_EXT_memory_object_win32") || !context->hasExtension("GL_EXT_semaphore_win32")) +#else if (!context->hasExtension("GL_EXT_memory_object_fd") || !context->hasExtension("GL_EXT_semaphore_fd")) +#endif return false; auto gl = context->functions(); auto extra = context->extraFunctions(); @@ -119,16 +131,30 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); auto deleteMemory = glFunction(context, "glDeleteMemoryObjectsEXT"); auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); +#ifdef _WIN32 + auto importMemory = glFunction(context, "glImportMemoryWin32HandleEXT"); +#else auto importMemory = glFunction(context, "glImportMemoryFdEXT"); +#endif auto textureMemory = glFunction(context, "glTexStorageMem2DEXT"); auto createSemaphore = glFunction(context, "glGenSemaphoresEXT"); auto deleteSemaphore = glFunction(context, "glDeleteSemaphoresEXT"); +#ifdef _WIN32 + auto importSemaphore = glFunction(context, "glImportSemaphoreWin32HandleEXT"); +#else auto importSemaphore = glFunction(context, "glImportSemaphoreFdEXT"); +#endif auto waitSemaphore = glFunction(context, "glWaitSemaphoreEXT"); auto signalSemaphore = glFunction(context, "glSignalSemaphoreEXT"); +#ifdef _WIN32 + auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryWin32HandleKHR")); + auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreWin32HandleKHR")); + if (!getMemoryHandle || !getSemaphoreHandle || vk->queue_family_comp_index < 0) return false; +#else auto getMemoryFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryFdKHR")); auto getSemaphoreFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreFdKHR")); if (!getMemoryFd || !getSemaphoreFd || vk->queue_family_comp_index < 0) return false; +#endif // Construct all fallible GL resources before transferring image ownership. QOpenGLShaderProgram program; if (!program.addShaderFromSourceCode(QOpenGLShader::Vertex, @@ -143,7 +169,11 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { VkSemaphore handoff[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; VkCommandPool pool = VK_NULL_HANDLE; VkFence fence = VK_NULL_HANDLE; +#ifdef _WIN32 + HANDLE memoryHandle = nullptr, semaphoreHandle = nullptr; +#else int memoryFd = -1, semaphoreFd = -1; +#endif auto cleanup = [&]() { if (vertexArray) { extra->glBindVertexArray(previousVertexArray); @@ -161,15 +191,27 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { for (auto semaphore : handoff) if (semaphore) vkDestroySemaphore(device, semaphore, nullptr); stage("delete Vulkan resources"); +#ifdef _WIN32 + if (memoryHandle) CloseHandle(memoryHandle); + if (semaphoreHandle) CloseHandle(semaphoreHandle); +#else if (memoryFd >= 0) close(memoryFd); if (semaphoreFd >= 0) close(semaphoreFd); +#endif }; bool locked = false; try { +#ifdef _WIN32 + VkMemoryGetWin32HandleInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR}; + memoryInfo.memory = image->mem[0]; + memoryInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT; + check(getMemoryHandle(device, &memoryInfo, &memoryHandle), "Export Vulkan image memory"); +#else VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; memoryInfo.memory = image->mem[0]; memoryInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; check(getMemoryFd(device, &memoryInfo, &memoryFd), "Export Vulkan image memory"); +#endif createMemory(1, &memory); VkMemoryDedicatedRequirements dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; VkMemoryRequirements2 requirements{VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated}; @@ -178,8 +220,16 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { vkGetImageMemoryRequirements2(device, &requirementInfo, &requirements); GLint useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; memoryParameter(memory, GL_DEDICATED_MEMORY_OBJECT_EXT, &useDedicated); +#ifdef _WIN32 + importMemory(memory, image->size[0], GL_HANDLE_TYPE_OPAQUE_WIN32_EXT, memoryHandle); + // Unlike file descriptors, Win32 imports retain the resource, not + // ownership of the application's handle. + CloseHandle(memoryHandle); + memoryHandle = nullptr; +#else importMemory(memory, image->size[0], GL_HANDLE_TYPE_OPAQUE_FD_EXT, memoryFd); memoryFd = -1; // GL consumes imported descriptors. +#endif stage("import memory"); gl->glGenTextures(1, &texture); gl->glBindTexture(GL_TEXTURE_2D, texture); @@ -190,19 +240,33 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { textureMemory(GL_TEXTURE_2D, 1, GL_RGBA8, source->width, source->height, memory, image->offset[0]); stage("texture storage"); VkExportSemaphoreCreateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; +#ifdef _WIN32 + exportInfo.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; +#else exportInfo.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; +#endif VkSemaphoreCreateInfo semaphoreInfo{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &exportInfo}; // Separate directions: reusing one binary semaphore lets Vulkan's // acquire consume its own release signal before GL can wait on it. createSemaphore(2, semaphores); for (int direction = 0; direction < 2; ++direction) { check(vkCreateSemaphore(device, &semaphoreInfo, nullptr, &handoff[direction]), "Create presentation semaphore"); +#ifdef _WIN32 + VkSemaphoreGetWin32HandleInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR}; + handleInfo.semaphore = handoff[direction]; + handleInfo.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; + check(getSemaphoreHandle(device, &handleInfo, &semaphoreHandle), "Export presentation semaphore"); + importSemaphore(semaphores[direction], GL_HANDLE_TYPE_OPAQUE_WIN32_EXT, semaphoreHandle); + CloseHandle(semaphoreHandle); + semaphoreHandle = nullptr; +#else VkSemaphoreGetFdInfoKHR fdInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; fdInfo.semaphore = handoff[direction]; fdInfo.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; check(getSemaphoreFd(device, &fdInfo, &semaphoreFd), "Export presentation semaphore"); importSemaphore(semaphores[direction], GL_HANDLE_TYPE_OPAQUE_FD_EXT, semaphoreFd); semaphoreFd = -1; +#endif } if (gl->glGetError() != GL_NO_ERROR) throw std::runtime_error("OpenGL external memory import failed"); const uint32_t family = vk->queue_family_comp_index; diff --git a/src/Qt/VulkanOpenGLPresenterMac.mm b/src/Qt/VulkanOpenGLPresenterMac.mm new file mode 100644 index 000000000..e0d5b0acb --- /dev/null +++ b/src/Qt/VulkanOpenGLPresenterMac.mm @@ -0,0 +1,172 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#define GL_SILENCE_DEPRECATION +#include "VulkanOpenGLPresenter.h" +#include "VideoRenderer.h" +#include "../VulkanFrameImage.h" +#include "../VulkanCompositor.h" +#include "../MacVulkanInterop.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#import +#import +#import +#import +#include +extern "C" { +#include +} +namespace openshot { +namespace { +bool SameDevice(CGLContextObj context, uint64_t registry) { + if (!context || !registry) return false; + GLint current = 0; + if (CGLGetParameter(context, kCGLCPCurrentRendererID, ¤t) != kCGLNoError) return false; + CGLRendererInfoObj info = nullptr; + GLint count = 0; + if (CGLQueryRendererInfo(0xffffffffu, &info, &count) != kCGLNoError) return false; + bool matched = false; + for (GLint i = 0; i < count; ++i) { + GLint renderer = 0, low = 0, high = 0; + CGLDescribeRenderer(info, i, kCGLRPRendererID, &renderer); + if ((renderer & kCGLRendererIDMatchingMask) != (current & kCGLRendererIDMatchingMask)) continue; + if (CGLDescribeRenderer(info, i, kCGLRPRegistryIDLow, &low) == kCGLNoError && + CGLDescribeRenderer(info, i, kCGLRPRegistryIDHigh, &high) == kCGLNoError) + matched |= ((uint64_t(uint32_t(high)) << 32) | uint32_t(low)) == registry; + } + CGLDestroyRendererInfo(info); + return matched; +} +} +struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { + QPointer widget; + std::shared_ptr frame; + std::shared_ptr display_surface; + bool fallback_delivered = false; + uint64_t native_draws = 0, fallback_frames = 0; + explicit Impl(QWidget* value) : widget(value) { + value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); + } + ~Impl() override { if (widget) widget->setProperty("openshot_gpu_presenter", QVariant()); } + void fallback() { + if (!fallback_delivered && frame && widget) { + fallback_delivered = true; ++fallback_frames; display_surface.reset(); + const QImage image = *frame->GetImage(); + QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + } + } + void PresentFrame(const std::shared_ptr& value) override { + frame = value; display_surface.reset(); fallback_delivered = false; + if (frame && !frame->GetDeferredImage()) fallback(); + if (widget) { + if (frame && frame->GetDeferredImage() && widget->metaObject()->indexOfMethod("gpu_presented(int)") >= 0) + QMetaObject::invokeMethod(widget, "gpu_presented", Qt::DirectConnection, + Q_ARG(int, static_cast(frame->number))); + widget->update(); + } + } + bool draw(int x, int y, int width, int height) { + auto* context = QOpenGLContext::currentContext(); + auto cgl = CGLGetCurrentContext(); + if (!context || !cgl || !frame || !widget || fallback_delivered) return false; + auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(frame->GetDeferredImage()); + if (!surface) return false; + auto* fc = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); + if (fc->sw_format != AV_PIX_FMT_RGBA || !IsMacVulkanFrame(surface->GetFrame())) { + surface = std::dynamic_pointer_cast(RenderVulkanImage( + surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); + if (!surface) return false; + } + const AVFrame* source = surface->GetFrame(); + auto pixel = static_cast(MacVulkanPixelBuffer(source)); + if (!pixel || !SameDevice(cgl, MacVulkanDeviceRegistryID(source))) return false; + display_surface = surface; + WaitMacVulkanFrame(source); + auto io = CVPixelBufferGetIOSurface(pixel); + if (!io) return false; + auto* gl = context->functions(); + auto* extra = context->extraFunctions(); + const bool core = context->format().profile() == QSurfaceFormat::CoreProfile; + QOpenGLShaderProgram program; + const char* vertex = core ? + "#version 150\nin vec2 vertex;out vec2 uv;void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}" : + "#version 120\nattribute vec2 vertex;varying vec2 uv;void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}"; + // CGL's supported 32-bit IOSurface view is BGRA. Our compositor stores + // RGBA bytes, so undo the view's red/blue interpretation in the sampler. + const char* fragment = core ? + "#version 150\nuniform sampler2DRect video;uniform vec2 size;in vec2 uv;out vec4 color;void main(){color=texture(video,uv*size).bgra;}" : + "#version 120\n#extension GL_ARB_texture_rectangle : enable\nuniform sampler2DRect video;uniform vec2 size;varying vec2 uv;void main(){gl_FragColor=texture2DRect(video,uv*size).bgra;}"; + if (!program.addShaderFromSourceCode(QOpenGLShader::Vertex, vertex) || + !program.addShaderFromSourceCode(QOpenGLShader::Fragment, fragment) || !program.link()) + throw std::runtime_error(program.log().toStdString()); + GLuint texture = 0, buffer = 0, vao = 0; + GLint oldViewport[4], oldVao = 0; + gl->glGetIntegerv(GL_VIEWPORT, oldViewport); + if (core) gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &oldVao); + auto cleanup = [&]() { + if (buffer) gl->glDeleteBuffers(1, &buffer); + if (vao) { extra->glBindVertexArray(oldVao); extra->glDeleteVertexArrays(1, &vao); } + if (texture) gl->glDeleteTextures(1, &texture); + gl->glViewport(oldViewport[0], oldViewport[1], oldViewport[2], oldViewport[3]); + }; + try { + gl->glActiveTexture(GL_TEXTURE0); + gl->glGenTextures(1, &texture); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, texture); + const auto imported = CGLTexImageIOSurface2D(cgl, GL_TEXTURE_RECTANGLE, GL_RGBA8, + source->width, source->height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, io, 0); + if (imported != kCGLNoError) throw std::runtime_error("CGL shared IOSurface import failed"); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + if (core) { extra->glGenVertexArrays(1, &vao); extra->glBindVertexArray(vao); } + const float vertices[] = {-1,-1, 1,-1, -1,1, 1,1}; + gl->glGenBuffers(1, &buffer); gl->glBindBuffer(GL_ARRAY_BUFFER, buffer); + gl->glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + if (!program.bind()) throw std::runtime_error("Bind Mac presentation shader"); + program.setUniformValue("video", 0); + program.setUniformValue("size", QVector2D(source->width, source->height)); + const int attribute = program.attributeLocation("vertex"); + program.enableAttributeArray(attribute); program.setAttributeBuffer(attribute, GL_FLOAT, 0, 2); + gl->glViewport(x, y, width, height); + gl->glDisable(GL_DEPTH_TEST); gl->glDisable(GL_SCISSOR_TEST); + gl->glDisable(GL_STENCIL_TEST); gl->glDisable(GL_BLEND); + gl->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + gl->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + // The next renderer may reuse the retained surface after PresentFrame. + // Complete GL sampling before releasing our reference to that IOSurface. + gl->glFinish(); + const auto error = gl->glGetError(); + program.disableAttributeArray(attribute); program.release(); + gl->glBindBuffer(GL_ARRAY_BUFFER, 0); + if (error != GL_NO_ERROR) throw std::runtime_error("Mac shared-texture presentation failed"); + cleanup(); + return true; + } catch (...) { cleanup(); throw; } + } +}; +VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) : impl(new Impl(reinterpret_cast(widget))) {} +VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; +bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { + try { if (impl->draw(x,y,width,height)) { ++impl->native_draws; return true; } } + catch (const std::exception& error) { qWarning() << "Mac Vulkan presentation fallback:" << error.what(); } + impl->fallback(); return false; +} +int VulkanOpenGLPresenter::Width() const { return impl->frame ? impl->frame->GetWidth() : 0; } +int VulkanOpenGLPresenter::Height() const { return impl->frame ? impl->frame->GetHeight() : 0; } +void VulkanOpenGLPresenter::Clear() { impl->frame.reset(); impl->display_surface.reset(); } +uint64_t VulkanOpenGLPresenter::NativeDraws() const { return impl->native_draws; } +uint64_t VulkanOpenGLPresenter::FallbackFrames() const { return impl->fallback_frames; } +void VulkanOpenGLPresenter::Present(std::shared_ptr frame) { + if (QThread::currentThread() == impl->thread()) impl->PresentFrame(frame); + else QMetaObject::invokeMethod(impl.get(), [receiver=impl.get(), frame]() { receiver->PresentFrame(frame); }, Qt::QueuedConnection); +} +} diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 292a0f27e..0b2e1585d 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -3,6 +3,9 @@ #include "VulkanCompositor.h" #include "VulkanEncodeConverter.h" #include "VulkanFrameImage.h" +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif #include #include #include @@ -102,6 +105,7 @@ class Pipeline { bool busy=false; std::vector views; FramePtr target; + std::vector inputs; }; std::array ring; int next_slot=0; @@ -189,6 +193,7 @@ class Pipeline { for(auto view:slot.views) vkDestroyImageView(device,view,nullptr); slot.views.clear(); slot.target.reset(); + slot.inputs.clear(); } VkImageView View(VkImage image,VkFormat format,VkImageAspectFlags aspect){ VkImageViewCreateInfo vi{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; @@ -222,10 +227,14 @@ class Pipeline { throw std::runtime_error("Encoder target cannot be written by a Vulkan shader"); AvCheck(av_frame_ref(output.get(),target),"Retain encoder target"); }else{ +#ifdef __APPLE__ + output.reset(AllocateMacVulkanFrame(device_ref,width,height,encode_nv12)); +#else AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12); const int allocated=av_hwframe_get_buffer(pool,output.get(),0); av_buffer_unref(&pool); AvCheck(allocated,"Allocate compositor output"); +#endif } AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); output->colorspace=encode_nv12?AVCOL_SPC_SMPTE170M:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; @@ -255,6 +264,11 @@ class Pipeline { p.coeffs[0]=full?1.0:255.0/219.0;p.extra[1]=full?0.0:16.0/255.0;p.extra[2]=128.0/255.0; p.coeffs[1]=2*(1-kr)*cscale;p.coeffs[2]=-2*kb*(1-kb)/(1-kr-kb)*cscale; p.coeffs[3]=-2*kr*(1-kr)/(1-kr-kb)*cscale;p.extra[0]=2*(1-kb)*cscale; + // Retire before locking inputs: releasing the last imported surface can + // wait on that frame's synchronization mutex. + Slot& slot=ring[next_slot]; + next_slot=(next_slot+1)%kSlots; + Retire(slot); // Lock in stable address order, avoiding deadlocks when frames are shared. std::vector all{src,output.get()}; if(back && back->data[0]!=src->data[0]) all.push_back(back); @@ -266,18 +280,19 @@ class Pipeline { const bool separate_nv12 = p.modes[0]==1 && sf->img[1]!=VK_NULL_HANDLE && sf->img[1]!=sf->img[0]; if(separate_nv12) plane_count=2; const auto* svc=reinterpret_cast(src_fc->hwctx); - Slot& slot=ring[next_slot]; - next_slot=(next_slot+1)%kSlots; - Retire(slot); auto& owned_views=slot.views; VkCommandBuffer cmd=slot.cmd; const VkDescriptorSet set=slot.set; try { // Ordinary outputs use our shared pools. An encoder's pool can // disappear when its writer closes, before this cached pipeline. - if(target){ - slot.target.reset(av_frame_clone(output.get())); - if(!slot.target) throw std::bad_alloc(); + slot.target.reset(av_frame_clone(output.get())); + if(!slot.target) throw std::bad_alloc(); + for(const AVFrame* input_frame : {src, back}) { + if (!input_frame) continue; + FramePtr retained(av_frame_clone(input_frame)); + if (!retained) throw std::bad_alloc(); + slot.inputs.push_back(std::move(retained)); } std::array views{}; auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ @@ -357,6 +372,8 @@ class Pipeline { locks.clear(); return std::make_shared(output.get(),!encode_nv12); }catch(...){ + // Imported surface destructors acquire the same frame mutexes. + locks.clear(); Retire(slot); throw; } @@ -407,6 +424,9 @@ std::shared_ptr ConvertVulkanToNV12(std::shared_ptr lock(shared.mutex); for(auto& item:shared.pools) av_buffer_unref(&item.second); diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 03cf7ef6b..8b2173015 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -3,6 +3,9 @@ #include "VulkanFrameImage.h" #include #include "VulkanCompositor.h" +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif #include #include #include @@ -63,6 +66,9 @@ int VulkanFrameImage::GetWidth() const { return impl->frame->width; } int VulkanFrameImage::GetHeight() const { return impl->frame->height; } const AVFrame* VulkanFrameImage::GetFrame() const { return impl->frame.get(); } int64_t VulkanFrameImage::GetBytes() const { +#ifdef __APPLE__ + if (IsMacVulkanFrame(impl->frame.get())) return MacVulkanFrameBytes(impl->frame.get()); +#endif const auto* frame = reinterpret_cast(impl->frame->data[0]); int64_t bytes = 0; for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i) { @@ -83,7 +89,12 @@ int64_t VulkanFrameImage::GetBytes() const { QImage VulkanFrameImage::Materialize() const { std::lock_guard lock(impl->mutex); auto cpu = AllocateFrame(); - Check(av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0), "Read back Vulkan frame"); +#ifdef __APPLE__ + if (IsMacVulkanFrame(impl->frame.get())) + Check(ReadMacVulkanFrame(cpu.get(), impl->frame.get()), "Read back Mac Vulkan frame"); + else +#endif + Check(av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0), "Read back Vulkan frame"); readback_count.fetch_add(1,std::memory_order_relaxed); Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); QImage result(GetWidth(), GetHeight(), QImage::Format_RGBA8888); @@ -113,6 +124,13 @@ VulkanFrameProcessor::VulkanFrameProcessor() : impl(new Impl) {} VulkanFrameProcessor::~VulkanFrameProcessor() = default; void VulkanFrameProcessor::Reset() {} std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* frame, int width, int height, bool high_quality) { +#ifdef __APPLE__ + OwnedFrame imported; + if (frame && frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { + imported.reset(ImportVideoToolboxFrame(frame)); + frame = imported.get(); + } +#endif Validate(frame); if (width <= 0 || height <= 0) throw std::invalid_argument("Invalid Vulkan output dimensions"); // Preserve the existing FFmpegReader conversion matrix (SWS_CS_DEFAULT / BT.601). diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe836b8e5..30c7032b3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -99,10 +99,7 @@ set(OPENSHOT_TESTS ) if(ENABLE_GPU_FRAME) - list(APPEND OPENSHOT_TESTS VulkanCompositor) - if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - list(APPEND OPENSHOT_TESTS VulkanPresentation) - endif() + list(APPEND OPENSHOT_TESTS VulkanCompositor VulkanPresentation) endif() # The SIGPIPE crash-handler test is POSIX-only. Do not create an empty diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 591102910..867d4b12e 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -1,3 +1,6 @@ +#ifdef __APPLE__ +#include "MacVulkanInterop.h" +#endif // Copyright (c) 2026 OpenShot Studios, LLC // SPDX-License-Identifier: LGPL-3.0-or-later #include "openshot_catch.h" @@ -125,12 +128,14 @@ TEST_CASE("Native Vulkan export keeps decoded and composited frames on the GPU", writer.SetVideoOptions(true,"h264_vulkan",openshot::Fraction(24,1),width,height,openshot::Fraction(1,1),false,false,1000000); writer.Open(); REQUIRE(writer.info.vcodec=="h264_vulkan"); // CPU fallback must not pass this test. + std::shared_ptr expected_frame; const auto readbacks=openshot::VulkanFrameReadbackCount(); for(int n=1;n<=24;++n){ auto frame=timeline.GetFrame(n); REQUIRE(frame->GetDeferredImage()); writer.WriteFrame(frame); REQUIRE(frame->GetDeferredImage()); + if (n == 12) expected_frame = frame; } writer.Close(); CHECK(openshot::VulkanFrameReadbackCount()==readbacks); @@ -278,7 +283,11 @@ TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan] CHECK(planes->img[0]!=planes->img[1]); Frame downloaded(av_frame_alloc()); REQUIRE(downloaded); +#ifdef __APPLE__ + REQUIRE(openshot::ReadMacVulkanFrame(downloaded.get(), encoded_surface->GetFrame()) >= 0); +#else REQUIRE(av_hwframe_transfer_data(downloaded.get(), encoded_surface->GetFrame(), 0) >= 0); +#endif REQUIRE(downloaded->format == AV_PIX_FMT_NV12); const int y = downloaded->data[0][4 * downloaded->linesize[0] + 4]; const int u = downloaded->data[1][2 * downloaded->linesize[1] + 4]; @@ -428,3 +437,67 @@ TEST_CASE("Vulkan high quality reader resizing uses bicubic antialiasing", "[vul if(size!=QSize(32,32)) CHECK(high_error expected_frame; + const auto readbacks=openshot::VulkanFrameReadbackCount(); + for(int n=1;n<=24;++n){ + auto frame=timeline.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + writer.WriteFrame(frame); + REQUIRE(frame->GetDeferredImage()); + if (n == 12) expected_frame = frame; + } + writer.Close(); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + CHECK(base_reader.HardwareDecodeSuccessful()); + CHECK(overlay_reader.HardwareDecodeSuccessful()); + timeline.Close(); + openshot::FFmpegReader verify(path); + verify.Open(); + CHECK(verify.info.width==width); CHECK(verify.info.height==height); + CHECK(verify.info.video_length==24); + auto image=verify.GetFrame(12)->GetImage(); + REQUIRE(image); CHECK(image->width()==width); CHECK(image->height()==height); + const auto expected = expected_frame->GetImage(); + double error = 0, brightness = 0; + int samples = 0; + for (int y = 8; y < height; y += 16) { + for (int x = 8; x < width; x += 16) { + const auto a = expected->pixelColor(x, y), b = image->pixelColor(x, y); + error += std::abs(a.red()-b.red()) + std::abs(a.green()-b.green()) + std::abs(a.blue()-b.blue()); + brightness += a.red() + a.green() + a.blue(); + samples += 3; + } + } + CHECK(brightness / samples > 10); + CHECK(error / samples < 20); + verify.Close(); + } +} +#endif diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 51a21d6db..afb195cf3 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -48,11 +48,17 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ return; } const QByteArray platform = qgetenv("QT_QPA_PLATFORM"); +#ifdef __linux__ if ((platform.isEmpty() || platform.startsWith("xcb")) && !qEnvironmentVariableIsSet("DISPLAY") && !qEnvironmentVariableIsSet("WAYLAND_DISPLAY")) { WARN("Skipping Vulkan test: presentation needs a display with OpenGL"); return; } +#endif + if (platform == "offscreen" || platform == "minimal") { + WARN("Skipping Vulkan test: presentation needs a native OpenGL display"); + return; + } Frames data; if (openshot::CreateSharedVulkanDevice(&data.device) < 0) { WARN("Skipping Vulkan test: no Vulkan device is available"); @@ -89,11 +95,17 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ widget.show(); QCoreApplication::processEvents(); auto* context = widget.context(); - if (!context || !context->hasExtension("GL_EXT_memory_object_fd") || - !context->hasExtension("GL_EXT_semaphore_fd")) { + if (!context +#ifdef _WIN32 + || !context->hasExtension("GL_EXT_memory_object_win32") || !context->hasExtension("GL_EXT_semaphore_win32") +#elif !defined(__APPLE__) + || !context->hasExtension("GL_EXT_memory_object_fd") || !context->hasExtension("GL_EXT_semaphore_fd") +#endif + ) { WARN("Skipping Vulkan test: OpenGL lacks Vulkan memory/semaphore interop"); return; } + const auto readbacks = openshot::VulkanFrameReadbackCount(); renderer.paint(frame); for (int i = 0; i < 20; ++i) { QCoreApplication::processEvents(); @@ -109,6 +121,9 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ REQUIRE(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); } REQUIRE(widget.nativeDraw); + CHECK(widget.presenter.NativeDraws() >= 64); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); REQUIRE(frame->GetDeferredImage()); REQUIRE_FALSE(image.isNull()); if (qEnvironmentVariableIsSet("OPENSHOT_PRESENTATION_CAPTURE")) From bd78acd456a232e6cafd448680539891af3465c7 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 30 Oct 2026 16:49:08 -0500 Subject: [PATCH 20/50] Report GPU capabilities and native presentation fallback reasons --- bindings/python/CMakeLists.txt | 5 ++++ src/FFmpegReader.cpp | 10 ++++++++ src/FFmpegWriter.cpp | 5 ++++ src/GpuDevices.cpp | 27 +++++++++++++++++--- src/GpuDevices.h | 2 ++ src/Qt/VulkanOpenGLPresenter.cpp | 40 +++++++++++++++++++++--------- src/Qt/VulkanOpenGLPresenter.h | 3 +++ src/Qt/VulkanOpenGLPresenterMac.mm | 26 +++++++++++++------ tests/VulkanPresentation.cpp | 4 +++ 9 files changed, 99 insertions(+), 23 deletions(-) diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index ef43d543f..55fc3c4cf 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -89,6 +89,11 @@ if (CMAKE_VERSION VERSION_GREATER 3.20) endif() ### Add the SWIG interface file (which defines all the SWIG methods) +# Older UseSWIG versions do not track transitive %include headers. Regenerate +# the wrapper when GPU diagnostics change, including in incremental builds. +set(SWIG_MODULE_pyopenshot_EXTRA_DEPS + "${PROJECT_SOURCE_DIR}/src/GpuDevices.h" + "${PROJECT_SOURCE_DIR}/src/Qt/VulkanOpenGLPresenter.h") swig_add_library(pyopenshot LANGUAGE python SOURCES openshot.i) if (POLICY CMP0078) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 2a5c65941..ed8dfeca1 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -1797,17 +1797,27 @@ bool FFmpegReader::GetAVFrame() { #if USE_HW_ACCEL if (hw_de_on && hw_de_supported && !force_sw_decode) { #ifdef OPENSHOT_ENABLE_GPU_FRAME + const bool previously_decoded_on_gpu = hw_decode_succeeded; hw_decode_succeeded = hw_decode_succeeded || (next_frame2->format == AV_PIX_FMT_VULKAN #ifdef __APPLE__ || next_frame2->format == AV_PIX_FMT_VIDEOTOOLBOX #endif ); + if (!previously_decoded_on_gpu && hw_decode_succeeded) + Logger::Instance()->Log("GPU decode confirmed: " + info.vcodec + "; " + + (next_frame2->format == AV_PIX_FMT_VULKAN ? "Vulkan" : "VideoToolbox") + "; " + path, + Logger::LevelInfo); #else hw_decode_succeeded = true; #endif } #endif packet_status.video_decoded++; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (packet_status.video_decoded == 1 && !hw_decode_succeeded) + Logger::Instance()->Log("GPU decode not observed: CPU video frame received; codec=" + + info.vcodec + "; " + path, Logger::LevelInfo); +#endif // Allocate image (align 32 for simd) AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format); diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index e81eeb79a..03e977b18 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -150,6 +150,11 @@ void FFmpegWriter::Open() { } #else open_video(oc, video_st); +#endif +#ifdef OPENSHOT_ENABLE_GPU_FRAME + Logger::Instance()->Log("GPU export setup: encoder=" + info.vcodec + + "; hardware encoding configured=" + (hw_en_on && hw_en_supported ? "yes" : "no") + + "; frame interop is checked when frames arrive", Logger::LevelInfo); #endif } if (info.has_audio && audio_st) diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 33201638b..7cfca273d 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -14,6 +14,7 @@ extern "C" { #include #include #include +#include } namespace openshot { namespace { @@ -72,6 +73,9 @@ GpuDevice Describe(VkPhysicalDevice device) { GpuDevice gpu; gpu.name = basic.deviceName; + gpu.vulkan_api = std::to_string(VK_API_VERSION_MAJOR(basic.apiVersion)) + "." + + std::to_string(VK_API_VERSION_MINOR(basic.apiVersion)) + "." + + std::to_string(VK_API_VERSION_PATCH(basic.apiVersion)); switch (basic.vendorID) { case 0x10DE: gpu.vendor = "NVIDIA"; break; case 0x1002: gpu.vendor = "AMD"; break; @@ -105,6 +109,8 @@ GpuDevice Describe(VkPhysicalDevice device) { vkGetPhysicalDeviceQueueFamilyProperties(device, &families, queues.data()); gpu.video_decode = std::any_of(queues.begin(), queues.end(), [](const VkQueueFamilyProperties& q) { return q.queueFlags & VK_QUEUE_VIDEO_DECODE_BIT_KHR; }); + gpu.video_encode = std::any_of(queues.begin(), queues.end(), + [](const VkQueueFamilyProperties& q) { return q.queueFlags & VK_QUEUE_VIDEO_ENCODE_BIT_KHR; }); return gpu; } @@ -145,7 +151,10 @@ VkPhysicalDevice PhysicalDevice(AVBufferRef* device) { // Try opening an encoder at a tiny size on the given hardware device. bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format) { const AVCodec* codec = avcodec_find_encoder_by_name(name); - if (!codec) return false; + if (!codec) { + Logger::Instance()->Log(std::string("GPU encoder probe: ") + name + " is not compiled into FFmpeg", Logger::LevelDebug); + return false; + } AVCodecContext* context = avcodec_alloc_context3(codec); if (!context) return false; context->width = 640; @@ -173,8 +182,15 @@ bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format context->pix_fmt = AV_PIX_FMT_NV12; context->hw_device_ctx = av_buffer_ref(device); } - if (context->hw_frames_ctx || context->hw_device_ctx) - opened = avcodec_open2(context, codec, nullptr) >= 0; + if (context->hw_frames_ctx || context->hw_device_ctx) { + const int result = avcodec_open2(context, codec, nullptr); + opened = result >= 0; + if (result < 0) { + char message[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, message, sizeof(message)); + Logger::Instance()->Log(std::string("GPU encoder probe failed: ") + name + ": " + message, Logger::LevelDebug); + } + } avcodec_free_context(&context); return opened; } @@ -203,6 +219,7 @@ int CreateSharedVulkanDevice(AVBufferRef** output) { #endif int result = AVERROR(ENODEV); for (const auto& device : VulkanDeviceCandidates()) { + Logger::Instance()->Log("Vulkan device initialization: " + (device.empty() ? std::string("default") : device), Logger::LevelInfo); AVDictionary* options = nullptr; #ifdef __APPLE__ av_dict_set(&options, "device_extensions", "VK_EXT_metal_objects", 0); @@ -212,6 +229,10 @@ int CreateSharedVulkanDevice(AVBufferRef** output) { av_dict_free(&options); if (result >= 0) break; + char message[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, message, sizeof(message)); + Logger::Instance()->Log("Vulkan device initialization failed: " + device + ": " + message + + " (" + std::to_string(result) + ")", Logger::LevelWarning); } if (result < 0) { Logger::Instance()->Log("No usable Vulkan GPU; decoding on the CPU", Logger::LevelWarning); diff --git a/src/GpuDevices.h b/src/GpuDevices.h index d85c0fd9a..24663523d 100644 --- a/src/GpuDevices.h +++ b/src/GpuDevices.h @@ -14,9 +14,11 @@ struct GpuDevice { std::string vendor; ///< "NVIDIA", "AMD", "Intel", or the PCI vendor id std::string type; ///< "discrete", "integrated", "virtual", "software" or "other" std::string driver; ///< Driver name and version, when the driver reports them + std::string vulkan_api; ///< Vulkan API version advertised by the physical device std::string uuid; ///< Stable id (hex), the same one CUDA reports for the card std::string render_node; ///< Linux DRM render node such as /dev/dri/renderD128, if known bool video_decode = false; ///< Has a Vulkan video decode queue + bool video_encode = false; ///< Has a Vulkan video encode queue (not codec-specific support) bool active = false; ///< The card OpenShot renders with }; /// Every Vulkan device on this system, including software ones. Empty without Vulkan. diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 3e38f1377..d9c42a233 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -48,6 +48,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { bool fallback_delivered = false; uint64_t native_draws = 0; uint64_t fallback_frames = 0; + std::string last_failure; + bool unavailable(const std::string& reason) { last_failure = reason; return false; } explicit Impl(QWidget* value) : widget(value) { value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); } @@ -58,7 +60,10 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { frame = value; display_surface.reset(); fallback_delivered = false; - if (frame && !frame->GetDeferredImage()) fallback(); + if (frame && !frame->GetDeferredImage()) { + last_failure = "Frame is CPU-backed (decode, rendering, or an effect required CPU pixels)"; + fallback(); + } if (widget) { if (frame && frame->GetDeferredImage() && widget->metaObject()->indexOfMethod("gpu_presented(int)") >= 0) QMetaObject::invokeMethod(widget, "gpu_presented", Qt::DirectConnection, @@ -85,16 +90,18 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { lastStage = now; }; auto context = QOpenGLContext::currentContext(); - if (!context || !frame || !widget || fallback_delivered) return false; + if (!context) return unavailable("No current Qt OpenGL context"); + if (!frame || !widget) return unavailable("No video frame or preview widget"); + if (fallback_delivered) return false; auto authoritative = frame->GetDeferredImage(); - if (!authoritative) return false; + if (!authoritative) return unavailable("Frame has no deferred GPU surface"); auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(authoritative); - if (!surface) return false; + if (!surface) return unavailable("Frame cannot be presented as a Vulkan RGBA surface"); auto inputContext = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); if (inputContext->sw_format != AV_PIX_FMT_RGBA) { surface = std::dynamic_pointer_cast(RenderVulkanImage( surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); - if (!surface) return false; + if (!surface) return unavailable("Frame cannot be presented as a Vulkan RGBA surface"); } display_surface = surface; stage("surface"); @@ -106,13 +113,13 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { auto image = reinterpret_cast(source->data[0]); // Multi-plane decode surfaces must first go through the RGBA compositor. if (fc->sw_format != AV_PIX_FMT_RGBA || vfc->format[0] != VK_FORMAT_R8G8B8A8_UNORM || image->img[1]) - return false; + return unavailable("Unsupported Vulkan image format for OpenGL sharing"); #ifdef _WIN32 if (!context->hasExtension("GL_EXT_memory_object_win32") || !context->hasExtension("GL_EXT_semaphore_win32")) #else if (!context->hasExtension("GL_EXT_memory_object_fd") || !context->hasExtension("GL_EXT_semaphore_fd")) #endif - return false; + return unavailable("OpenGL driver lacks external memory or semaphore sharing extensions"); auto gl = context->functions(); auto extra = context->extraFunctions(); auto getUuid = glFunction(context, "glGetUnsignedBytei_vEXT"); @@ -127,7 +134,12 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { getUuid(GL_DEVICE_UUID_EXT, i, uuid); sameDevice = sameDevice || std::memcmp(uuid, identity.deviceUUID, VK_UUID_SIZE) == 0; } - if (!sameDevice) return false; + if (!sameDevice) { + const auto* renderer = gl->glGetString(GL_RENDERER); + return unavailable(std::string("Qt/OpenGL and Vulkan selected different GPUs (OpenGL: ") + + (renderer ? reinterpret_cast(renderer) : "unknown") + + "; Vulkan: " + properties.properties.deviceName + ")"); + } auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); auto deleteMemory = glFunction(context, "glDeleteMemoryObjectsEXT"); auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); @@ -149,18 +161,20 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { #ifdef _WIN32 auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryWin32HandleKHR")); auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreWin32HandleKHR")); - if (!getMemoryHandle || !getSemaphoreHandle || vk->queue_family_comp_index < 0) return false; + if (!getMemoryHandle || !getSemaphoreHandle || vk->queue_family_comp_index < 0) + return unavailable("Vulkan device lacks Win32 memory/semaphore export or a compute queue"); #else auto getMemoryFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryFdKHR")); auto getSemaphoreFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreFdKHR")); - if (!getMemoryFd || !getSemaphoreFd || vk->queue_family_comp_index < 0) return false; + if (!getMemoryFd || !getSemaphoreFd || vk->queue_family_comp_index < 0) + return unavailable("Vulkan device lacks file-descriptor memory/semaphore export or a compute queue"); #endif // Construct all fallible GL resources before transferring image ownership. QOpenGLShaderProgram program; if (!program.addShaderFromSourceCode(QOpenGLShader::Vertex, "attribute vec2 vertex; varying vec2 uv; void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}") || !program.addShaderFromSourceCode(QOpenGLShader::Fragment, - "uniform sampler2D video; varying vec2 uv; void main(){gl_FragColor=texture2D(video,uv);}") || !program.link()) return false; + "uniform sampler2D video; varying vec2 uv; void main(){gl_FragColor=texture2D(video,uv);}") || !program.link()) return unavailable("OpenGL presentation shader: " + program.log().toStdString()); stage("shader"); VkDevice device = vk->act_dev; GLuint memory = 0, texture = 0, semaphores[2] = {0, 0}, buffer = 0, vertexArray = 0; @@ -382,8 +396,9 @@ VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { try { - if (impl->draw(x, y, width, height)) { ++impl->native_draws; return true; } + if (impl->draw(x, y, width, height)) { impl->last_failure.clear(); ++impl->native_draws; return true; } } catch (const std::exception& error) { + impl->last_failure = error.what(); qWarning() << "Vulkan presentation fallback:" << error.what(); } impl->fallback(); @@ -397,6 +412,7 @@ void VulkanOpenGLPresenter::Clear() { } uint64_t VulkanOpenGLPresenter::NativeDraws() const { return impl->native_draws; } uint64_t VulkanOpenGLPresenter::FallbackFrames() const { return impl->fallback_frames; } +std::string VulkanOpenGLPresenter::FallbackReason() const { return impl->last_failure; } void VulkanOpenGLPresenter::Present(std::shared_ptr frame) { if (QThread::currentThread() == impl->thread()) { impl->PresentFrame(frame); diff --git a/src/Qt/VulkanOpenGLPresenter.h b/src/Qt/VulkanOpenGLPresenter.h index b22543046..2f02225d3 100644 --- a/src/Qt/VulkanOpenGLPresenter.h +++ b/src/Qt/VulkanOpenGLPresenter.h @@ -4,6 +4,7 @@ #define OPENSHOT_VULKAN_OPENGL_PRESENTER_H #include #include +#include namespace openshot { class Frame; /// GUI-thread adapter for a QOpenGLWidget. Draw requires its current GL context. @@ -17,6 +18,8 @@ class VulkanOpenGLPresenter { void Clear(); uint64_t NativeDraws() const; uint64_t FallbackFrames() const; + /// Most recent native presentation failure; empty after a successful draw. + std::string FallbackReason() const; void Present(std::shared_ptr frame); private: struct Impl; diff --git a/src/Qt/VulkanOpenGLPresenterMac.mm b/src/Qt/VulkanOpenGLPresenterMac.mm index e0d5b0acb..0ca33ecb8 100644 --- a/src/Qt/VulkanOpenGLPresenterMac.mm +++ b/src/Qt/VulkanOpenGLPresenterMac.mm @@ -51,6 +51,8 @@ bool SameDevice(CGLContextObj context, uint64_t registry) { std::shared_ptr display_surface; bool fallback_delivered = false; uint64_t native_draws = 0, fallback_frames = 0; + std::string last_failure; + bool unavailable(const std::string& reason) { last_failure = reason; return false; } explicit Impl(QWidget* value) : widget(value) { value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); } @@ -64,7 +66,10 @@ void fallback() { } void PresentFrame(const std::shared_ptr& value) override { frame = value; display_surface.reset(); fallback_delivered = false; - if (frame && !frame->GetDeferredImage()) fallback(); + if (frame && !frame->GetDeferredImage()) { + last_failure = "Frame is CPU-backed (decode, rendering, or an effect required CPU pixels)"; + fallback(); + } if (widget) { if (frame && frame->GetDeferredImage() && widget->metaObject()->indexOfMethod("gpu_presented(int)") >= 0) QMetaObject::invokeMethod(widget, "gpu_presented", Qt::DirectConnection, @@ -75,22 +80,26 @@ void PresentFrame(const std::shared_ptr& value) override { bool draw(int x, int y, int width, int height) { auto* context = QOpenGLContext::currentContext(); auto cgl = CGLGetCurrentContext(); - if (!context || !cgl || !frame || !widget || fallback_delivered) return false; + if (!context || !cgl) return unavailable("No current Qt/CGL OpenGL context"); + if (!frame || !widget) return unavailable("No video frame or preview widget"); + if (fallback_delivered) return false; auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(frame->GetDeferredImage()); - if (!surface) return false; + if (!surface) return unavailable("Frame cannot be presented as a shared Mac Vulkan surface"); auto* fc = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); if (fc->sw_format != AV_PIX_FMT_RGBA || !IsMacVulkanFrame(surface->GetFrame())) { surface = std::dynamic_pointer_cast(RenderVulkanImage( surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); - if (!surface) return false; + if (!surface) return unavailable("Frame cannot be presented as a shared Mac Vulkan surface"); } const AVFrame* source = surface->GetFrame(); auto pixel = static_cast(MacVulkanPixelBuffer(source)); - if (!pixel || !SameDevice(cgl, MacVulkanDeviceRegistryID(source))) return false; + if (!pixel) return unavailable("Vulkan frame has no shared Core Video pixel buffer"); + if (!SameDevice(cgl, MacVulkanDeviceRegistryID(source))) + return unavailable("Qt/OpenGL and Vulkan/Metal selected different GPUs"); display_surface = surface; WaitMacVulkanFrame(source); auto io = CVPixelBufferGetIOSurface(pixel); - if (!io) return false; + if (!io) return unavailable("Core Video pixel buffer has no IOSurface"); auto* gl = context->functions(); auto* extra = context->extraFunctions(); const bool core = context->format().profile() == QSurfaceFormat::CoreProfile; @@ -156,8 +165,8 @@ bool draw(int x, int y, int width, int height) { VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) : impl(new Impl(reinterpret_cast(widget))) {} VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { - try { if (impl->draw(x,y,width,height)) { ++impl->native_draws; return true; } } - catch (const std::exception& error) { qWarning() << "Mac Vulkan presentation fallback:" << error.what(); } + try { if (impl->draw(x,y,width,height)) { impl->last_failure.clear(); ++impl->native_draws; return true; } } + catch (const std::exception& error) { impl->last_failure = error.what(); qWarning() << "Mac Vulkan presentation fallback:" << error.what(); } impl->fallback(); return false; } int VulkanOpenGLPresenter::Width() const { return impl->frame ? impl->frame->GetWidth() : 0; } @@ -165,6 +174,7 @@ bool draw(int x, int y, int width, int height) { void VulkanOpenGLPresenter::Clear() { impl->frame.reset(); impl->display_surface.reset(); } uint64_t VulkanOpenGLPresenter::NativeDraws() const { return impl->native_draws; } uint64_t VulkanOpenGLPresenter::FallbackFrames() const { return impl->fallback_frames; } +std::string VulkanOpenGLPresenter::FallbackReason() const { return impl->last_failure; } void VulkanOpenGLPresenter::Present(std::shared_ptr frame) { if (QThread::currentThread() == impl->thread()) impl->PresentFrame(frame); else QMetaObject::invokeMethod(impl.get(), [receiver=impl.get(), frame]() { receiver->PresentFrame(frame); }, Qt::QueuedConnection); diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index afb195cf3..79d4d419f 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -123,6 +123,7 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ REQUIRE(widget.nativeDraw); CHECK(widget.presenter.NativeDraws() >= 64); CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(widget.presenter.FallbackReason().empty()); CHECK(openshot::VulkanFrameReadbackCount() == readbacks); REQUIRE(frame->GetDeferredImage()); REQUIRE_FALSE(image.isNull()); @@ -133,4 +134,7 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); CHECK(image.pixelColor(5, 5) == QColor(Qt::green)); + widget.presenter.Clear(); + CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); + CHECK_FALSE(widget.presenter.FallbackReason().empty()); } From 2eadbfa6118187aff47ddcf59c7efacfd6472281 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 31 Oct 2026 16:29:41 -0500 Subject: [PATCH 21/50] Track GPU work and render CPU-origin images with Vulkan Keep playback and export counters separate, attribute CPU readbacks, and preserve measurement resets across in-flight work. Cache bounded GPU uploads for image sources so scaling, compositing, and presentation can stay on GPU. --- bindings/python/CMakeLists.txt | 3 ++ bindings/python/openshot.i | 2 + src/CMakeLists.txt | 2 + src/Clip.cpp | 35 +++++++++++++- src/FFmpegReader.cpp | 7 +++ src/FFmpegWriter.cpp | 15 ++++++ src/FFmpegWriter.h | 1 + src/GpuDevices.cpp | 15 ++++++ src/GpuDevices.h | 3 ++ src/GpuMetrics.cpp | 74 +++++++++++++++++++++++++++++ src/GpuMetrics.h | 40 ++++++++++++++++ src/Timeline.cpp | 4 ++ src/Timeline.h | 1 + src/VulkanCompositor.cpp | 1 + src/VulkanFrameImage.cpp | 69 +++++++++++++++++++++++++++ src/VulkanFrameImage.h | 4 ++ tests/CMakeLists.txt | 1 + tests/GpuMetrics.cpp | 57 ++++++++++++++++++++++ tests/VulkanCompositor.cpp | 87 ++++++++++++++++++++++++++++++++++ tests/VulkanPresentation.cpp | 19 ++++++++ 20 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 src/GpuMetrics.cpp create mode 100644 src/GpuMetrics.h create mode 100644 tests/GpuMetrics.cpp diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 55fc3c4cf..662dae963 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -93,6 +93,9 @@ endif() # the wrapper when GPU diagnostics change, including in incremental builds. set(SWIG_MODULE_pyopenshot_EXTRA_DEPS "${PROJECT_SOURCE_DIR}/src/GpuDevices.h" + "${PROJECT_SOURCE_DIR}/src/GpuMetrics.h" + "${PROJECT_SOURCE_DIR}/src/Timeline.h" + "${PROJECT_SOURCE_DIR}/src/FFmpegWriter.h" "${PROJECT_SOURCE_DIR}/src/Qt/VulkanOpenGLPresenter.h") swig_add_library(pyopenshot LANGUAGE python SOURCES openshot.i) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index 1fb38805c..c7b3cfddb 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -116,6 +116,7 @@ class QWidget; #include "VulkanFrameImage.h" #include "GpuDevices.h" #endif +#include "GpuMetrics.h" #include "Settings.h" #include "TimelineBase.h" #include "Timeline.h" @@ -533,6 +534,7 @@ namespace openshot { void ReleaseVulkanMemory(); uint64_t VulkanFrameReadbackCou %template(GpuDeviceVector) std::vector; %template(StringVector) std::vector; #endif +%include "GpuMetrics.h" %include "Settings.h" %include "TimelineBase.h" %include "Qt/VideoCacheThread.h" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 642f2bca3..648be5876 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -436,6 +436,8 @@ message(STATUS "Found Qt${QT_VERSION_MAJOR} ${Qt${QT_VERSION_MAJOR}Core_VERSION_ set(QT_VERSION_STR ${Qt${QT_VERSION_MAJOR}Core_VERSION_STRING} CACHE STRING "Qt version linked with" FORCE) mark_as_advanced(QT_VERSION_STR) +target_sources(openshot PRIVATE GpuMetrics.cpp) + ################### FFMPEG ##################### # Find FFmpeg libraries (used for video encoding / decoding) find_package(FFmpeg REQUIRED diff --git a/src/Clip.cpp b/src/Clip.cpp index 68b12b557..7f2b95bf7 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -10,6 +10,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include #include "Clip.h" @@ -30,6 +31,7 @@ #include "effects/AudioVisualization.h" #include +#include #include #include #include @@ -792,6 +794,7 @@ std::shared_ptr Clip::GetOrCreateFrame(int64_t number, bool enable_time) "number", number, "clip_frame_number", clip_frame_number); // Attempt to get a frame (but this could fail if a reader has just been closed) + if (reader->info.has_single_image) gpu_metrics::Record(gpu_metrics::ImageSource); auto reader_frame = reader->GetFrame(clip_frame_number); if (reader_frame) { // Override frame # (due to time-mapping might change it) @@ -1390,6 +1393,7 @@ void Clip::apply_background(std::shared_ptr frame, try { auto result = RenderVulkanImage(source, background, background_frame->GetWidth(), background_frame->GetHeight(), QTransform(), 1.0, clear); + gpu_metrics::Record(gpu_metrics::RenderGpu); background_frame->AddDeferredImage(result); if (update_frame_image) frame->AddDeferredImage(result); return; @@ -1399,6 +1403,8 @@ void Clip::apply_background(std::shared_ptr frame, } } #endif + gpu_metrics::Record(gpu_metrics::RenderCpu); + gpu_metrics::Reason gpu_reason("Composite"); // Add background canvas std::shared_ptr background_canvas = background_frame->GetImage(); QPainter painter(background_canvas.get()); @@ -1421,9 +1427,13 @@ void Clip::apply_effects(std::shared_ptr frame, int64_t timeline_frame_nu { // Apply the effect to this frame if (effect->info.apply_before_clip && before_keyframes) { + gpu_metrics::Reason gpu_reason("Effect"); effect->ProcessFrame(frame, frame->number); + if (effect->info.has_video) gpu_metrics::RecordRender(*frame); } else if (!effect->info.apply_before_clip && !before_keyframes) { + gpu_metrics::Reason gpu_reason("Effect"); effect->ProcessFrame(frame, frame->number); + if (effect->info.has_video) gpu_metrics::RecordRender(*frame); } } @@ -1442,6 +1452,23 @@ bool Clip::isNear(double a, double b) } #ifdef OPENSHOT_ENABLE_GPU_FRAME +namespace { +std::shared_ptr VulkanSource(const std::shared_ptr& frame) { + if (!VulkanRenderingEnabled()) return {}; + auto source = std::dynamic_pointer_cast(frame->GetDeferredImage()); + if (source || !frame->has_image_data) return source; + try { + auto uploaded = UploadVulkanImage(*frame->GetImage()); + frame->AddDeferredImage(uploaded); + return std::dynamic_pointer_cast(uploaded); + } catch (const std::exception& error) { + static std::atomic_flag warned = ATOMIC_FLAG_INIT; + if (!warned.test_and_set()) + Logger::Instance()->Log(std::string("GPU image upload unavailable; using CPU: ") + error.what(), Logger::LevelWarning); + return {}; + } +} +} bool Clip::apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr background_frame, QSize timeline_size) { if (!frame->has_image_data || composite != COMPOSITE_SOURCE_OVER || @@ -1454,7 +1481,7 @@ bool Clip::apply_vulkan_layer(std::shared_ptr frame, std::shared_ptr(timeline)->Effects().empty()) return false; - auto source = std::dynamic_pointer_cast(frame->GetDeferredImage()); + auto source = VulkanSource(frame); auto background = background_frame->GetDeferredImage(); QColor clear; if (!source || !(std::dynamic_pointer_cast(background) || background_frame->GetBlankColor(clear))) @@ -1463,6 +1490,7 @@ bool Clip::apply_vulkan_layer(std::shared_ptr frame, std::shared_ptrAddDeferredImage(RenderVulkanImage(source, background, timeline_size.width(), timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()), alpha.GetValue(frame->number), clear)); + gpu_metrics::Record(gpu_metrics::RenderGpu); return true; } catch (const std::exception& error) { Logger::Instance()->Log(std::string("Vulkan layer CPU fallback: ") + error.what()); @@ -1482,11 +1510,12 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { #ifdef OPENSHOT_ENABLE_GPU_FRAME if (composite == COMPOSITE_SOURCE_OVER && corner_radius.GetValue(frame->number) == 0.0 && display == FRAME_DISPLAY_NONE) { - if (auto source = std::dynamic_pointer_cast(frame->GetDeferredImage())) { + if (auto source = VulkanSource(frame)) { try { frame->AddDeferredImage(RenderVulkanImage(source, nullptr, timeline_size.width(), timeline_size.height(), get_transform(frame, timeline_size.width(), timeline_size.height()), alpha.GetValue(frame->number))); + gpu_metrics::Record(gpu_metrics::RenderGpu); return; } catch (const std::exception& error) { Logger::Instance()->Log(std::string("Vulkan transform CPU fallback: ") + error.what()); @@ -1495,6 +1524,8 @@ void Clip::apply_keyframes(std::shared_ptr frame, QSize timeline_size) { } #endif + gpu_metrics::Record(gpu_metrics::RenderCpu); + gpu_metrics::Reason gpu_reason("Transform"); // Get image from clip, and create transparent background image std::shared_ptr source_image = frame->GetImage(); std::shared_ptr background_canvas = std::make_shared(timeline_size.width(), diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index ed8dfeca1..5f8910a30 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -13,6 +13,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include // for std::this_thread::sleep_for #include // for std::chrono::milliseconds #include @@ -1812,6 +1813,10 @@ bool FFmpegReader::GetAVFrame() { #endif } #endif + const auto* metrics_format = av_pix_fmt_desc_get(static_cast(next_frame2->format)); + if (!info.has_single_image) + gpu_metrics::Record(metrics_format && (metrics_format->flags & AV_PIX_FMT_FLAG_HWACCEL) + ? gpu_metrics::DecodeGpu : gpu_metrics::DecodeCpu); packet_status.video_decoded++; #ifdef OPENSHOT_ENABLE_GPU_FRAME if (packet_status.video_decoded == 1 && !hw_decode_succeeded) @@ -2167,6 +2172,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { if (!vulkan_processor) vulkan_processor = std::make_shared(); auto f = CreateFrame(current_frame); f->AddDeferredImage(vulkan_processor->Process(pFrame, width, height, openshot::Settings::Instance()->HIGH_QUALITY_SCALING)); + gpu_metrics::Record(gpu_metrics::RenderGpu); if (ApplyOrientationMetadata() && std::fabs(source_rotation) >= 0.0001) Logger::Instance()->AppendDebugMethod("Vulkan reader: source orientation requires scaled CPU readback"); ApplyFrameOrientation(f); @@ -2233,6 +2239,7 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { } // Resize / Convert to RGB + gpu_metrics::Record(gpu_metrics::RenderCpu); const int scaled_lines = sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0, original_height, pFrameRGB->data, pFrameRGB->linesize); if (scaled_lines <= 0) { diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 03e977b18..dcb04d74c 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -13,6 +13,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include #include #include @@ -1137,6 +1138,11 @@ void FFmpegWriter::WriteHeader() { } // Add a frame to the queue waiting to be encoded. +void FFmpegWriter::WriteMeasuredFrame(ReaderBase* reader, int64_t number) { + gpu_metrics::Scope scope(true); + WriteFrame(reader->GetFrame(number)); +} + void FFmpegWriter::WriteFrame(std::shared_ptr frame) { // Check for open reader (or throw exception) if (!is_open) @@ -1188,6 +1194,7 @@ void FFmpegWriter::WriteFrameAt(std::shared_ptr frame, int64_t // Write all frames in the queue to the video file. void FFmpegWriter::write_frame(std::shared_ptr frame) { + gpu_metrics::Reason gpu_reason("Encoder"); // Flip writing flag is_writing = true; @@ -3010,6 +3017,7 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra av_frame_free(&hw_frame); throw ErrorEncodingVideo("Could not upload CPU frame to encoder", -1); } + gpu_metrics::Record(gpu_metrics::Upload); av_frame_copy_props(hw_frame, frame_final); } #endif // USE_HW_ACCEL @@ -3032,6 +3040,13 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr frame, AVFrame *fra { ret = avcodec_send_frame(video_codec_ctx, frame_final); } + if (ret >= 0) { +#if USE_HW_ACCEL + gpu_metrics::Record(hw_en_on && hw_en_supported ? gpu_metrics::EncodeGpu : gpu_metrics::EncodeCpu); +#else + gpu_metrics::Record(gpu_metrics::EncodeCpu); +#endif + } error_code = ret; if (ret < 0 ) { Logger::Instance()->AppendDebugMethod( diff --git a/src/FFmpegWriter.h b/src/FFmpegWriter.h index fa9603464..c05c66cb6 100644 --- a/src/FFmpegWriter.h +++ b/src/FFmpegWriter.h @@ -215,6 +215,7 @@ namespace openshot { void write_frame(std::shared_ptr frame); public: + void WriteMeasuredFrame(ReaderBase* reader, int64_t number); /// @brief Constructor for FFmpegWriter. /// Throws an exception on failure to open path. diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 7cfca273d..8ad4a2328 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -10,6 +10,7 @@ #include #include #include +#include extern "C" { #include #include @@ -256,6 +257,20 @@ GpuDevice ActiveGpuDevice() { return gpu; } +void ConfigureGpuEnvironment(const std::string& backend, const std::string& device) { +#ifdef _WIN32 + if (_putenv_s("OPENSHOT_RENDER_BACKEND", backend.c_str()) != 0 || + _putenv_s("OPENSHOT_VULKAN_DEVICE", device.c_str()) != 0) + throw std::runtime_error("Unable to apply native GPU environment"); +#else + const auto apply = [](const char* key, const std::string& value) { + return value.empty() ? unsetenv(key) : setenv(key, value.c_str(), 1); + }; + if (apply("OPENSHOT_RENDER_BACKEND", backend) != 0 || apply("OPENSHOT_VULKAN_DEVICE", device) != 0) + throw std::runtime_error("Unable to apply native GPU environment"); +#endif +} + std::vector ListGpuDevices() { const std::string active = ActiveGpuDevice().uuid; ListingInstance listing; diff --git a/src/GpuDevices.h b/src/GpuDevices.h index 24663523d..94922d9e9 100644 --- a/src/GpuDevices.h +++ b/src/GpuDevices.h @@ -21,6 +21,9 @@ struct GpuDevice { bool video_encode = false; ///< Has a Vulkan video encode queue (not codec-specific support) bool active = false; ///< The card OpenShot renders with }; +/// Apply before any device/worker initialization. Keep the native C runtime's +/// environment consistent with Python when Windows uses different runtimes. +void ConfigureGpuEnvironment(const std::string& backend, const std::string& device); /// Every Vulkan device on this system, including software ones. Empty without Vulkan. std::vector ListGpuDevices(); /// The card OpenShot renders with, choosing it on first use. Empty name when diff --git a/src/GpuMetrics.cpp b/src/GpuMetrics.cpp new file mode 100644 index 000000000..b000cd09e --- /dev/null +++ b/src/GpuMetrics.cpp @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" +#include "Frame.h" +#include "Json.h" +#include +#include +#include +#include +#include +namespace openshot { namespace gpu_metrics { +struct Counters { + std::array, Count> values{}; + std::atomic resident{0}, requests{0}; + std::array, 5> reasons{}; +}; +namespace { +std::shared_ptr sessions[2] = {std::make_shared(), std::make_shared()}; +thread_local Scope* current = nullptr; +thread_local const char* reason = "Other"; +const char* names[] = {"decode_gpu", "decode_cpu", "image_sources", "render_gpu", "render_cpu", + "readbacks", "uploads", "encode_gpu", "encode_cpu"}; +const char* reasons[] = {"Effect", "Transform", "Composite", "Encoder", "Other"}; +} +Scope::Scope(bool exporting) : previous(current), exceptions(std::uncaught_exceptions()) { + if (!current) { + counters = std::atomic_load(&sessions[exporting ? 1 : 0]); + current = this; + } +} +Scope::~Scope() { + if (counters) { + if (gpu && std::uncaught_exceptions() == exceptions) { + counters->requests.fetch_add(1, std::memory_order_relaxed); + if (!readback) counters->resident.fetch_add(1, std::memory_order_relaxed); + } + current = previous; + } +} +Reason::Reason(const char* name) : previous(reason) { reason = name; } +Reason::~Reason() { reason = previous; } +void Record(Event event) { + if (!current) return; + current->counters->values[event].fetch_add(1, std::memory_order_relaxed); + if (event == DecodeGpu || event == RenderGpu || event == EncodeGpu) current->gpu = true; + if (event == Readback) { + current->readback = true; + size_t index = 4; + for (size_t i = 0; i < 4; ++i) if (std::strcmp(reason, reasons[i]) == 0) { index = i; break; } + current->counters->reasons[index].fetch_add(1, std::memory_order_relaxed); + } +} +void RecordRender(const Frame& frame) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + Record(frame.GetDeferredImage() ? RenderGpu : RenderCpu); +#else + Record(RenderCpu); +#endif +} +} // namespace gpu_metrics +std::string GpuMetricsSnapshot(bool exporting) { + using namespace gpu_metrics; + auto counters = std::atomic_load(&sessions[exporting ? 1 : 0]); + Json::Value root(Json::objectValue); + for (int i = 0; i < Count; ++i) root[names[i]] = Json::UInt64(counters->values[i].load(std::memory_order_relaxed)); + root["copy_free"] = Json::UInt64(counters->resident.load(std::memory_order_relaxed)); + root["gpu_requests"] = Json::UInt64(counters->requests.load(std::memory_order_relaxed)); + for (int i = 0; i < 5; ++i) root["readback_reasons"][reasons[i]] = Json::UInt64(counters->reasons[i].load(std::memory_order_relaxed)); + root["definition"] = "Operation counts exclude cache hits; copy-free counts GPU work requests without GPU-to-CPU readbacks, not end-to-end host-free frames"; + return root.toStyledString(); +} +void ResetGpuMetrics(bool exporting) { + std::atomic_store(&gpu_metrics::sessions[exporting ? 1 : 0], std::make_shared()); +} +} diff --git a/src/GpuMetrics.h b/src/GpuMetrics.h new file mode 100644 index 000000000..bcdcb1839 --- /dev/null +++ b/src/GpuMetrics.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_GPU_METRICS_H +#define OPENSHOT_GPU_METRICS_H +#include +#include +namespace openshot { +class Frame; +/// Bounded, operation counters. Export and playback have independent generations. +std::string GpuMetricsSnapshot(bool exporting = false); +void ResetGpuMetrics(bool exporting = false); +#ifndef SWIG +namespace gpu_metrics { +enum Event { DecodeGpu, DecodeCpu, ImageSource, RenderGpu, RenderCpu, + Readback, Upload, EncodeGpu, EncodeCpu, Count }; +/// Scope follows synchronous timeline work; unrelated background readers are ignored. +struct Counters; +class Scope { + Scope* previous; + std::shared_ptr counters; + bool gpu = false, readback = false; + int exceptions; + friend void Record(Event event); +public: + explicit Scope(bool exporting); + ~Scope(); + Scope(const Scope&) = delete; + Scope& operator=(const Scope&) = delete; +}; +class Reason { + const char* previous; +public: + explicit Reason(const char* name); + ~Reason(); +}; +void Record(Event event); +void RecordRender(const Frame& frame); +} +#endif +} +#endif diff --git a/src/Timeline.cpp b/src/Timeline.cpp index 51271af47..971aec481 100644 --- a/src/Timeline.cpp +++ b/src/Timeline.cpp @@ -10,6 +10,7 @@ // // SPDX-License-Identifier: LGPL-3.0-or-later +#include "GpuMetrics.h" #include "Timeline.h" #include "PreviewSize.h" @@ -605,7 +606,9 @@ std::shared_ptr Timeline::apply_effects(std::shared_ptr frame, int "does_effect_intersect", does_effect_intersect); // Apply the effect to this frame + gpu_metrics::Reason gpu_reason("Effect"); frame = effect->ProcessFrame(frame, effect_frame_number); + if (effect->info.has_video) gpu_metrics::RecordRender(*frame); } } // end effect loop @@ -985,6 +988,7 @@ bool Timeline::isEqual(double a, double b) // Get an openshot::Frame object for a specific frame number of this reader. std::shared_ptr Timeline::GetFrame(int64_t requested_frame) { + gpu_metrics::Scope gpu_scope(gpu_metrics_export); // Adjust out of bounds frame number if (requested_frame < 1) requested_frame = 1; diff --git a/src/Timeline.h b/src/Timeline.h index b5d270c78..7da1d2b2f 100644 --- a/src/Timeline.h +++ b/src/Timeline.h @@ -224,6 +224,7 @@ namespace openshot { void InvalidateCacheForClip(const openshot::Clip* clip); public: + bool gpu_metrics_export = false; // Set once before this timeline starts serving export work. /// @brief Constructor for the timeline (which configures the default frame properties) /// @param width The image width of generated openshot::Frame objects diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 0b2e1585d..5785cc897 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -424,6 +424,7 @@ std::shared_ptr ConvertVulkanToNV12(std::shared_ptr #include #include "VulkanCompositor.h" #ifdef __APPLE__ @@ -96,6 +99,7 @@ QImage VulkanFrameImage::Materialize() const { #endif Check(av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0), "Read back Vulkan frame"); readback_count.fetch_add(1,std::memory_order_relaxed); + gpu_metrics::Record(gpu_metrics::Readback); Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); QImage result(GetWidth(), GetHeight(), QImage::Format_RGBA8888); if (result.isNull()) throw std::bad_alloc(); @@ -143,4 +147,69 @@ std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* f matrix.scale(double(width) / frame->width, double(height) / frame->height); return RenderVulkanImage(source, nullptr, width, height, matrix, 1.0, Qt::transparent, high_quality); } +namespace { +struct UploadCache { + struct Entry { std::shared_ptr surface; int64_t bytes; uint64_t age; }; + std::mutex mutex; + std::map entries; + int64_t bytes = 0; + uint64_t clock = 0; +}; +UploadCache& Uploads() { static UploadCache cache; return cache; } +struct BufferDeleter { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; +using OwnedBuffer = std::unique_ptr; +} +std::shared_ptr UploadVulkanImage(const QImage& image) { + if (image.isNull()) throw std::invalid_argument("Cannot upload an empty image"); + auto& cache = Uploads(); + const std::lock_guard lock(cache.mutex); + const auto key = image.cacheKey(); + auto found = cache.entries.find(key); + if (found != cache.entries.end()) { + found->second.age = ++cache.clock; + return found->second.surface; + } + AVBufferRef* device_ptr = nullptr; + Check(CreateSharedVulkanDevice(&device_ptr), "Create image upload device"); + OwnedBuffer device(device_ptr); + OwnedBuffer pool(av_hwframe_ctx_alloc(device.get())); + if (!pool) throw std::bad_alloc(); + auto* context = reinterpret_cast(pool->data); + context->format = AV_PIX_FMT_VULKAN; + context->sw_format = AV_PIX_FMT_RGBA; + context->width = image.width(); context->height = image.height(); + auto* vk = reinterpret_cast(context->hwctx); + vk->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + Check(av_hwframe_ctx_init(pool.get()), "Initialize image upload pool"); + QImage rgba = image.convertToFormat(QImage::Format_RGBA8888_Premultiplied); + auto cpu = AllocateFrame(); + auto gpu = AllocateFrame(); + cpu->format = AV_PIX_FMT_RGBA; cpu->width = rgba.width(); cpu->height = rgba.height(); + cpu->data[0] = const_cast(rgba.constBits()); + cpu->linesize[0] = rgba.bytesPerLine(); + Check(av_hwframe_get_buffer(pool.get(), gpu.get(), 0), "Allocate uploaded image"); + Check(av_hwframe_transfer_data(gpu.get(), cpu.get(), 0), "Upload CPU image"); + gpu_metrics::Record(gpu_metrics::Upload); + auto surface = std::make_shared(gpu.get(), true); + const int64_t size = surface->GetBytes(); + constexpr int64_t budget = 64 * 1024 * 1024; + if (size <= budget) { + while (!cache.entries.empty() && (cache.bytes + size > budget || cache.entries.size() >= 16)) { + auto oldest = std::min_element(cache.entries.begin(), cache.entries.end(), + [](const auto& a, const auto& b) { return a.second.age < b.second.age; }); + cache.bytes -= oldest->second.bytes; + cache.entries.erase(oldest); + } + cache.entries.emplace(key, UploadCache::Entry{surface, size, ++cache.clock}); + cache.bytes += size; + } + return surface; +} +void ReleaseVulkanUploads() { + auto& cache = Uploads(); + const std::lock_guard lock(cache.mutex); + cache.entries.clear(); cache.bytes = 0; +} + } diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h index 94d12f5cb..85255d55c 100644 --- a/src/VulkanFrameImage.h +++ b/src/VulkanFrameImage.h @@ -29,6 +29,10 @@ class VulkanFrameImage final : public FrameImage { struct Impl; std::unique_ptr impl; }; +/// Upload CPU pixels once per QImage cache key. Bounded cache; returned surfaces +/// retain their own GPU storage. Mutable QImage access invalidates its cache key. +std::shared_ptr UploadVulkanImage(const QImage& image); +void ReleaseVulkanUploads(); /// Reuses the scaling pipeline across frames. Outputs own independent storage. class VulkanFrameProcessor { public: diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 30c7032b3..76313ae03 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,6 +34,7 @@ target_link_libraries(openshot-benchmark openshot) ### TEST SOURCE FILES ### set(OPENSHOT_TESTS + GpuMetrics AdaptivePreroll AudioDeviceManager AudioRecorder diff --git a/tests/GpuMetrics.cpp b/tests/GpuMetrics.cpp new file mode 100644 index 000000000..237c4c793 --- /dev/null +++ b/tests/GpuMetrics.cpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "openshot_catch.h" +#include "GpuMetrics.h" +#include "Json.h" +#include +using namespace openshot; +using namespace openshot::gpu_metrics; +namespace { +Json::Value snapshot(bool exporting = false) { + Json::Reader reader; + Json::Value value; + REQUIRE(reader.parse(GpuMetricsSnapshot(exporting), value)); + return value; +} +} +TEST_CASE("GPU metrics isolate work scopes and exports", "[GpuMetrics]") { + ResetGpuMetrics(); ResetGpuMetrics(true); + Record(DecodeCpu); // Unrelated reader/thumbnail work is not playback. + { Scope scope(false); Record(DecodeGpu); Record(RenderGpu); } + { Scope scope(true); Record(EncodeGpu); Record(Upload); } + REQUIRE(snapshot()["decode_cpu"].asUInt64() == 0); + REQUIRE(snapshot()["decode_gpu"].asUInt64() == 1); + REQUIRE(snapshot()["encode_gpu"].asUInt64() == 0); + REQUIRE(snapshot(true)["encode_gpu"].asUInt64() == 1); + REQUIRE(snapshot()["copy_free"].asUInt64() == 1); +} +TEST_CASE("GPU metrics count nested work once and attribute readbacks", "[GpuMetrics]") { + ResetGpuMetrics(); + { Scope outer(false); Record(RenderGpu); + { Scope nested(false); Reason reason("Effect"); Record(Readback); } + } + auto value = snapshot(); + REQUIRE(value["gpu_requests"].asUInt64() == 1); + REQUIRE(value["copy_free"].asUInt64() == 0); + REQUIRE(value["readback_reasons"]["Effect"].asUInt64() == 1); +} +TEST_CASE("GPU metric reset excludes old in-flight work", "[GpuMetrics]") { + ResetGpuMetrics(); + { Scope old(false); Record(RenderGpu); ResetGpuMetrics(); Record(Readback); } + REQUIRE(snapshot()["gpu_requests"].asUInt64() == 0); + REQUIRE(snapshot()["readbacks"].asUInt64() == 0); + { Scope fresh(false); Record(RenderGpu); } + REQUIRE(snapshot()["copy_free"].asUInt64() == 1); +} +TEST_CASE("GPU counters support parallel requests", "[GpuMetrics]") { + ResetGpuMetrics(); + auto work = [] { for (int i = 0; i < 1000; ++i) { Scope scope(false); Record(DecodeGpu); } }; + std::thread first(work), second(work); first.join(); second.join(); + REQUIRE(snapshot()["decode_gpu"].asUInt64() == 2000); + REQUIRE(snapshot()["gpu_requests"].asUInt64() == 2000); +} + +TEST_CASE("Failed GPU work is not a copy-free success", "[GpuMetrics]") { + ResetGpuMetrics(); + try { Scope scope(false); Record(DecodeGpu); throw 1; } catch (int) {} + REQUIRE(snapshot()["gpu_requests"].asUInt64() == 0); +} diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 867d4b12e..3ea31e3e7 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -8,6 +8,7 @@ #include "VulkanFrameImage.h" #include "VulkanEncodeConverter.h" #include "GpuDevices.h" +#include "GpuMetrics.h" #include "FFmpegReader.h" #include "FFmpegWriter.h" #include @@ -501,3 +502,89 @@ TEST_CASE("VideoToolbox export shares decoder compositor and encoder surfaces", } } #endif + +TEST_CASE("CPU image uploads cache pixels and preserve alpha ownership", "[vulkan][hardware][upload]") { + if (!VulkanAvailable()) return; + openshot::ReleaseVulkanUploads(); + QImage image(16, 16, QImage::Format_ARGB32_Premultiplied); + image.fill(QColor(200, 60, 20, 128)); + openshot::ResetGpuMetrics(); + std::shared_ptr first, second; + { + openshot::gpu_metrics::Scope scope(false); + first = openshot::UploadVulkanImage(image); + CHECK(openshot::UploadVulkanImage(image) == first); + image.fill(QColor(0, 255, 0, 255)); + second = openshot::UploadVulkanImage(image); + CHECK(second != first); + } + Json::Reader reader; + Json::Value metrics; + REQUIRE(reader.parse(openshot::GpuMetricsSnapshot(), metrics)); + CHECK(metrics["uploads"].asUInt64() == 2); + QTransform identity; + auto rendered = openshot::RenderVulkanImage(first, nullptr, 16, 16, identity, 1.0, Qt::transparent); + auto pixel = rendered->Materialize().pixelColor(8,8); + CHECK(std::abs(pixel.red() - 200) <= 2); + CHECK(std::abs(pixel.green() - 60) <= 2); + CHECK(std::abs(pixel.alpha() - 128) <= 1); + openshot::ReleaseVulkanUploads(); + CHECK(second->Materialize().pixelColor(8,8) == QColor(Qt::green)); + CHECK(openshot::UploadVulkanImage(image) != second); +} + +TEST_CASE("PNG clips upload once and keep animated transforms on Vulkan", "[vulkan][hardware][upload]") { + if (!VulkanAvailable()) return; + QTemporaryDir temporary; + REQUIRE(temporary.isValid()); + const auto path = temporary.filePath("artwork.png"); + QImage bitmap(64, 64, QImage::Format_ARGB32_Premultiplied); + bitmap.fill(QColor(0, 0, 255, 128)); + REQUIRE(bitmap.save(path)); + openshot::Clip clip(path.toStdString()); + clip.scale_x = openshot::Keyframe(0.6); + clip.scale_y = openshot::Keyframe(0.6); + clip.rotation = openshot::Keyframe(10); + openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + openshot::ResetGpuMetrics(); + std::shared_ptr result; + for (int i = 1; i <= 8; ++i) { + result = timeline.GetFrame(i); + REQUIRE(result->GetDeferredImage()); + } + Json::Reader reader; + Json::Value metrics; + REQUIRE(reader.parse(openshot::GpuMetricsSnapshot(), metrics)); + CHECK(metrics["uploads"].asUInt64() == 1); + CHECK(metrics["readbacks"].asUInt64() == 0); + CHECK(metrics["render_cpu"].asUInt64() == 0); + CHECK(metrics["render_gpu"].asUInt64() > 0); + CHECK(metrics["image_sources"].asUInt64() == 8); + CHECK(metrics["decode_cpu"].asUInt64() == 0); + CHECK(metrics["copy_free"].asUInt64() == 8); + openshot::Frame copy(*result); + const auto pixel = copy.GetImage()->pixelColor(64,64); + CHECK(std::abs(pixel.blue() - 128) <= 2); + CHECK(pixel.red() == 0); + CHECK(pixel.alpha() == 255); + timeline.Close(); +} + +TEST_CASE("Saved GPU preferences reach the native runtime environment", "[vulkan][settings]") { + struct Restore { + std::string backend, device; + ~Restore() { openshot::ConfigureGpuEnvironment(backend, device); } + } restore{std::getenv("OPENSHOT_RENDER_BACKEND") ? std::getenv("OPENSHOT_RENDER_BACKEND") : "", + std::getenv("OPENSHOT_VULKAN_DEVICE") ? std::getenv("OPENSHOT_VULKAN_DEVICE") : ""}; + openshot::ConfigureGpuEnvironment("cpu", "Example GPU"); + REQUIRE(std::getenv("OPENSHOT_RENDER_BACKEND")); + REQUIRE(std::getenv("OPENSHOT_VULKAN_DEVICE")); + CHECK(std::string(std::getenv("OPENSHOT_RENDER_BACKEND")) == "cpu"); + CHECK(std::string(std::getenv("OPENSHOT_VULKAN_DEVICE")) == "Example GPU"); + CHECK_FALSE(openshot::VulkanRenderingEnabled()); + openshot::ConfigureGpuEnvironment("vulkan", ""); + CHECK(openshot::VulkanRenderingEnabled()); + CHECK(std::getenv("OPENSHOT_VULKAN_DEVICE") == nullptr); +} diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 79d4d419f..e28609e96 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -3,6 +3,7 @@ #include "openshot_catch.h" #include "Frame.h" #include "VulkanFrameImage.h" +#include "VulkanCompositor.h" #include "GpuDevices.h" #include "Qt/VideoRenderer.h" #include "Qt/VulkanOpenGLPresenter.h" @@ -134,6 +135,24 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); CHECK(image.pixelColor(5, 5) == QColor(Qt::green)); + // CPU-origin artwork follows upload -> compositor -> native presentation. + QImage artwork(64, 64, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < 64; ++y) + for (int x = 0; x < 64; ++x) + artwork.setPixelColor(x, y, y < 32 ? Qt::red : Qt::blue); + auto uploaded = openshot::UploadVulkanImage(artwork); + frame->AddDeferredImage(openshot::RenderVulkanImage(uploaded, nullptr, 64, 64, QTransform(), 1.0, Qt::transparent)); + renderer.paint(frame); + for (int i = 0; i < 20; ++i) { + QCoreApplication::processEvents(); + QThread::msleep(5); + } + image = widget.grabFramebuffer(); + CHECK(widget.nativeDraw); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); + CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); widget.presenter.Clear(); CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); CHECK_FALSE(widget.presenter.FallbackReason().empty()); From 55db9cc87628fe9f7e2fc33fa3ad0ce787e71cdd Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sat, 31 Oct 2026 16:47:56 -0500 Subject: [PATCH 22/50] Use shared Mac surfaces for cached image uploads Normalize uploaded artwork into the IOSurface compositor path so CPU fallback uses the synchronized Metal readback bridge. Assert the Mac surface type and retain strict pixel checks with useful RGBA diagnostics. --- src/VulkanFrameImage.cpp | 8 +++++++- tests/VulkanCompositor.cpp | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 4eb2d7ffd..1ff8333ee 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -191,7 +191,13 @@ std::shared_ptr UploadVulkanImage(const QImage& image) { Check(av_hwframe_get_buffer(pool.get(), gpu.get(), 0), "Allocate uploaded image"); Check(av_hwframe_transfer_data(gpu.get(), cpu.get(), 0), "Upload CPU image"); gpu_metrics::Record(gpu_metrics::Upload); - auto surface = std::make_shared(gpu.get(), true); + std::shared_ptr surface = std::make_shared(gpu.get(), true); +#ifdef __APPLE__ + // Cache an IOSurface-backed image, so CPU fallback uses the synchronized + // Metal bridge instead of FFmpeg's generic MoltenVK staging-buffer readback. + surface = RenderVulkanImage(surface, nullptr, image.width(), image.height(), + QTransform(), 1.0, Qt::transparent); +#endif const int64_t size = surface->GetBytes(); constexpr int64_t budget = 64 * 1024 * 1024; if (size <= budget) { diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 3ea31e3e7..a93fab5b6 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -513,6 +513,11 @@ TEST_CASE("CPU image uploads cache pixels and preserve alpha ownership", "[vulka { openshot::gpu_metrics::Scope scope(false); first = openshot::UploadVulkanImage(image); +#ifdef __APPLE__ + auto mac_surface = std::dynamic_pointer_cast(first); + REQUIRE(mac_surface); + CHECK(openshot::IsMacVulkanFrame(mac_surface->GetFrame())); +#endif CHECK(openshot::UploadVulkanImage(image) == first); image.fill(QColor(0, 255, 0, 255)); second = openshot::UploadVulkanImage(image); @@ -529,7 +534,10 @@ TEST_CASE("CPU image uploads cache pixels and preserve alpha ownership", "[vulka CHECK(std::abs(pixel.green() - 60) <= 2); CHECK(std::abs(pixel.alpha() - 128) <= 1); openshot::ReleaseVulkanUploads(); - CHECK(second->Materialize().pixelColor(8,8) == QColor(Qt::green)); + const auto second_pixel = second->Materialize().pixelColor(8,8); + INFO("Uploaded opaque RGBA: " << second_pixel.red() << "," << second_pixel.green() + << "," << second_pixel.blue() << "," << second_pixel.alpha()); + CHECK(second_pixel == QColor(Qt::green)); CHECK(openshot::UploadVulkanImage(image) != second); } From 7001bb6f7055ce297a4c0d628d3899e38801dc89 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 00:01:25 -0500 Subject: [PATCH 23/50] Build Windows FFmpeg with NVENC, QSV, and AMF --- .gitlab-ci.yml | 4 + ci/windows/setup-ffmpeg-windows-final.sh | 65 ++++++++++++++ ci/windows/setup-ffmpeg-windows.sh | 7 ++ ci/windows/setup-hw-ffmpeg-windows.sh | 109 +++++++++++++++++++++++ ci/windows/smoke-ffmpeg-windows.sh | 20 +++++ src/FFmpegWriter.cpp | 5 +- src/Qt/VulkanOpenGLPresenter.cpp | 2 + tests/VulkanCompositor.cpp | 47 ++++++++++ tests/VulkanPresentation.cpp | 5 ++ 9 files changed, 263 insertions(+), 1 deletion(-) create mode 100755 ci/windows/setup-ffmpeg-windows-final.sh create mode 100755 ci/windows/setup-ffmpeg-windows.sh create mode 100755 ci/windows/setup-hw-ffmpeg-windows.sh create mode 100755 ci/windows/smoke-ffmpeg-windows.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d183274e..c2f802b89 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,6 +124,8 @@ windows-builder-x64: - $env:MSYSTEM = "MINGW64" - $env:FFMPEGDIR = "C:/msys64/mingw64" - if ((& "$env:FFMPEGDIR/bin/ffmpeg.exe" -version | Select-Object -First 1) -notmatch "^ffmpeg version 7[.]1[.]") { throw "FFmpeg 7.1 is required" } + - $gpuEncoders = (& "$env:FFMPEGDIR/bin/ffmpeg.exe" -hide_banner -encoders 2>&1 | Out-String) + - 'if ($LASTEXITCODE -ne 0) { throw "Could not enumerate FFmpeg encoders" }; foreach ($codec in @("h264_nvenc", "hevc_nvenc", "h264_qsv", "hevc_qsv", "h264_amf", "hevc_amf")) { if ($gpuEncoders -notmatch "\b$codec\b") { throw "Windows FFmpeg is missing required hardware encoder $codec" } }' - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw64" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x64" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x64" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -DENABLE_GPU_FRAME=ON @@ -199,6 +201,8 @@ windows-builder-x86: - $env:MSYSTEM = "MINGW32" - $env:FFMPEGDIR = "C:/msys64/mingw32" - if ((& "$env:FFMPEGDIR/bin/ffmpeg.exe" -version | Select-Object -First 1) -notmatch "^ffmpeg version 7[.]1[.]") { throw "FFmpeg 7.1 is required" } + - $gpuEncoders = (& "$env:FFMPEGDIR/bin/ffmpeg.exe" -hide_banner -encoders 2>&1 | Out-String) + - 'if ($LASTEXITCODE -ne 0) { throw "Could not enumerate FFmpeg encoders" }; foreach ($codec in @("h264_nvenc", "hevc_nvenc", "h264_qsv", "hevc_qsv", "h264_amf", "hevc_amf")) { if ($gpuEncoders -notmatch "\b$codec\b") { throw "Windows FFmpeg is missing required hardware encoder $codec" } }' - ffmpeg -hide_banner -devices | findstr /I "gdigrab" - ffmpeg -hide_banner -devices | findstr /I "dshow" - cmake -B build -S . -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"babl_DIR=C:/msys64/mingw32" -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR\build\install-x86" -D"OpenShotAudio_ROOT=$CI_PROJECT_DIR\build\install-x86" -D"OpenCV_DIR:PATH=$env:OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_AUTOGEN_PARALLEL=1 -G "MinGW Makefiles" -D"CMAKE_BUILD_TYPE:STRING=Release" -D"CMAKE_CXX_FLAGS=-m32 -msse2 -mfpmath=sse" -D"CMAKE_EXE_LINKER_FLAGS=-Wl,--large-address-aware" -D"CMAKE_C_FLAGS=-m32 -msse2 -mfpmath=sse" -DENABLE_GPU_FRAME=ON diff --git a/ci/windows/setup-ffmpeg-windows-final.sh b/ci/windows/setup-ffmpeg-windows-final.sh new file mode 100755 index 000000000..0e309f0c5 --- /dev/null +++ b/ci/windows/setup-ffmpeg-windows-final.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Run using C:\msys64\usr\bin\bash.exe. No pacman changes. +# Usage: bash setup-ffmpeg-windows.sh mingw64|mingw32 +set -euo pipefail +arch=${1:?Specify mingw64 or mingw32} +case "$arch" in mingw64) cpu=x86_64 ;; mingw32) cpu=x86 ;; *) exit 2 ;; esac +version=7.1.3 +root=/c/runner-setup +prefix=${PREFIX:-$root/ffmpeg-$version-$arch} +hw_prefix=${HW_PREFIX:-$root/hw-ffmpeg-$arch} +work=${BUILD_DIR:-$root/build-$version-$arch} +export PATH="$root/tools/bin:/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +export PKG_CONFIG_LIBDIR="/$arch/lib/pkgconfig:/$arch/share/pkgconfig" +export PKG_CONFIG_PATH="$hw_prefix/lib/pkgconfig:$prefix/lib/pkgconfig" +mkdir -p "$root/logs" "$prefix" "$work" +exec > >(tee -a "$root/logs/ffmpeg-$version-$arch.log") 2>&1 +trap 'echo "FAILED at line $LINENO"; exit 1' ERR +echo "Building $version $arch at $(date)" +gcc --version | head -1 +cd "$root" +if [ ! -f "FFmpeg-n$version.tar.gz" ]; then + curl --cacert /usr/ssl/cert.pem -fL --retry 2 "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n$version.tar.gz" -o "FFmpeg-n$version.tar.gz" +fi +printf '%s %s\n' e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb "FFmpeg-n$version.tar.gz" | sha256sum -c - +if [ ! -d "$root/FFmpeg-n$version" ]; then tar xzf "FFmpeg-n$version.tar.gz"; fi +cd "$work" +opts=(--prefix="$prefix" --target-os=mingw32 --arch="$cpu" + --enable-gpl --enable-version3 --enable-shared --disable-static --disable-doc + --enable-ffnvcodec --enable-nvenc --enable-nvdec --enable-cuvid --enable-amf --enable-libvpl + --extra-cflags="-I$hw_prefix/include" + --enable-x86asm --x86asmexe="$root/tools/bin/nasm.exe" --enable-libdav1d --extra-ldflags="-L$prefix/lib") +if [ "${WITH_VULKAN:-0}" = 1 ]; then + cflags="-I$hw_prefix/include -I$root/vulkan-$arch/include" + if [ "$arch" = mingw32 ]; then cflags="$cflags -msse2 -mfpmath=sse"; fi + opts+=(--enable-vulkan --extra-cflags="$cflags") +else + opts+=(--disable-vulkan) +fi +# Enable the existing codec libraries explicitly; configure reports incompatible ones. +for pair in libx264:x264 libx265:x265 libvpx:vpx libopus:opus libvorbis:vorbis libtheora:theoraenc libmp3lame:lame libwebp:libwebp libopenjpeg:libopenjp2 libass:libass libfreetype:freetype2 libfontconfig:fontconfig libfribidi:fribidi libsoxr:soxr; do + feature=${pair%%:*}; package=${pair#*:} + if pkg-config --exists "$package"; then opts+=("--enable-$feature"); fi +done +"$root/FFmpeg-n$version/configure" "${opts[@]}" +make -j"${JOBS:-6}" +make install +# The oneVPL dispatcher is an application dependency; vendor GPU drivers remain system supplied. +cp "$hw_prefix/bin/libvpl.dll" "$prefix/bin/" +mkdir -p "$prefix/share/vpl/licensing" +cp "$hw_prefix/share/vpl/licensing/"* "$prefix/share/vpl/licensing/" +export PATH="$prefix/bin:$hw_prefix/bin:$PATH" +"$prefix/bin/ffmpeg.exe" -version +"$prefix/bin/ffmpeg.exe" -hide_banner -f lavfi -i testsrc2=size=128x96:rate=5 -t 1 -c:v mpeg4 -y "$prefix/smoke-test.avi" +"$prefix/bin/ffprobe.exe" -v error -show_entries stream=codec_name,width,height "$prefix/smoke-test.avi" +cat > "$prefix/env.sh" < "$prefix/compiled-encoders.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$prefix/compiled-encoders.txt" || { echo "Missing required encoder: $codec"; exit 1; } +done +cp config.h config_components.h "$prefix/" +echo "SUCCESS: $prefix includes NVIDIA NVENC, Intel QSV, and AMD AMF" diff --git a/ci/windows/setup-ffmpeg-windows.sh b/ci/windows/setup-ffmpeg-windows.sh new file mode 100755 index 000000000..8b3646a44 --- /dev/null +++ b/ci/windows/setup-ffmpeg-windows.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +arch=${1:?Specify mingw64 or mingw32} +export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +export WITH_VULKAN=${WITH_VULKAN:-1} +bash "$(dirname "$0")/setup-hw-ffmpeg-windows.sh" "$arch" +exec bash "$(dirname "$0")/setup-ffmpeg-windows-final.sh" "$@" diff --git a/ci/windows/setup-hw-ffmpeg-windows.sh b/ci/windows/setup-hw-ffmpeg-windows.sh new file mode 100755 index 000000000..c42531e2d --- /dev/null +++ b/ci/windows/setup-hw-ffmpeg-windows.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Build pinned hardware encoder dependencies without changing the installed toolchain. +# Run with C:\msys64\usr\bin\bash.exe setup-hw-ffmpeg-windows.sh mingw64|mingw32. +set -euo pipefail +arch=${1:?Specify mingw64 or mingw32} +case "$arch" in mingw64|mingw32) ;; *) exit 2 ;; esac +root=/c/runner-setup +prefix=${HW_PREFIX:-$root/hw-ffmpeg-$arch} +export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +mkdir -p "$prefix" "$root/logs" +exec > >(tee -a "$root/logs/hw-ffmpeg-$arch.log") 2>&1 +trap 'echo "FAILED at line $LINENO"' ERR +fetch() { + local name=$1 url=$2 digest=$3 + if [ ! -f "$root/$name" ]; then curl --cacert /usr/ssl/cert.pem -fL --retry 2 "$url" -o "$root/$name.part"; mv "$root/$name.part" "$root/$name"; fi + printf '%s %s\n' "$digest" "$root/$name" | sha256sum -c - +} +fetch nv-codec-headers-n12.1.14.0.tar.gz https://github.com/FFmpeg/nv-codec-headers/archive/refs/tags/n12.1.14.0.tar.gz 2fefaa227d2a3b4170797796425a59d1dd2ed5fd231db9b4244468ba327acd0b +fetch libvpl-2.10.2.tar.gz https://github.com/intel/libvpl/archive/refs/tags/v2.10.2.tar.gz ad956ea7ecf14614325f59dfb44cc5ba08e2fcac373342d61c7db152ac651253 +for name in nv-codec-headers-n12.1.14.0 libvpl-2.10.2; do + source="$root/hw-sources/$name" + if [ ! -d "$source" ]; then mkdir -p "$source"; tar xzf "$root/$name.tar.gz" --strip-components=1 -C "$source"; fi +done +make -C "$root/hw-sources/nv-codec-headers-n12.1.14.0" install PREFIX="$prefix" +cmake -G 'MinGW Makefiles' -S "$root/hw-sources/libvpl-2.10.2" -B "$root/hw-vpl-build-$arch" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$(cygpath -m "$prefix")" \ + -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DBUILD_SHARED_LIBS=ON -DBUILD_TESTS=OFF \ + -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DINSTALL_EXAMPLE_CODE=OFF \ + -DBUILD_DISPATCHER_ONEVPL_EXPERIMENTAL=OFF +cmake --build "$root/hw-vpl-build-$arch" --parallel "${JOBS:-6}" +cmake --install "$root/hw-vpl-build-$arch" +# oneVPL installs Win32 DLLs under bin/x86; applications need the matching +# dispatcher directly beside FFmpeg, just like the x64 build. +if [ "$arch" = mingw32 ] && [ -f "$prefix/bin/x86/libvpl.dll" ]; then + cp "$prefix/bin/x86/libvpl.dll" "$prefix/bin/libvpl.dll" +fi +test -f "$prefix/bin/libvpl.dll" +# AMD AMF 1.4.35 headers, pinned to the release commit. No SDK binaries needed. +amf="$prefix/include/AMF" +mkdir -p "$amf" +while read -r digest header; do + mkdir -p "$amf/$(dirname "$header")" + if ! printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - >/dev/null 2>&1; then + curl --cacert /usr/ssl/cert.pem -fL --retry 2 \ + "https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/AMF/ba07d1b3c7c7ee0d846125cffc1d6d78a020c45a/amf/public/include/$header" -o "$amf/$header" + fi + printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - +done <<'AMF_HEADERS' +70afd5279be90e656c2ec270229c5a702a11435a670052370e96b270e6b7640f components/AMFXInput.h +27770ae77d2f7a16eca9ab1f8b9ce87cf0c0350425b950bdd39189c2c91a519a components/Ambisonic2SRenderer.h +05a9ebc32217e69b6c4659f4744dbda9e5a2792f3e56bc1bcf61c15dcfd46cab components/AudioCapture.h +bbe91a1e744685a36269d3b6af9cd36d9716ee748c8d0ed95255a6a42b46892f components/Capture.h +a3a89490d44adaffc2fa2cccc14313f8f65e5af7a6431d89d85bd80e36c628a1 components/ChromaKey.h +2f6887d14516ebdcf034d59e39235f5d9974533ddb0b04d685c0c3b6d397d3d6 components/ColorSpace.h +f3f18680d55a65fdf5e0351d3fde1dcd9bdd2d6caec268c8c915c4227ffa1edc components/Component.h +2479eb2a6174a1b63552c509abac1b701eddcab5a5eac5a7f4d9f40274d588b4 components/ComponentCaps.h +7c22337ff313320bd7fe20ef0fb00756735012367a5daa7c621725030f578863 components/CursorCapture.h +693c8e491f90eaffd226c1f6b2da37d3a5ef1d3c610e554647f7d0b699832727 components/DisplayCapture.h +9dde7e850c8bdecda82b0cee1b331294f6cc91b32b1d1504344a023cd4dda8af components/FFMPEGAudioConverter.h +92f50d789bc18d5ebd143b2925e124594a89110eef7efaa8189eb063bf2f3aa8 components/FFMPEGAudioDecoder.h +2493067de9b3f540ffaae8a6466e57ef75af4772f037a81785b11ee9511029f5 components/FFMPEGAudioEncoder.h +87a9ae9480c3af6210b1282d1768e36673fcc1ef4970afa6edaa74481cf0831b components/FFMPEGComponents.h +5d64b7d2d50ca17ba601885dcdb51bb16ef4dccf88323437967e8524f26194bc components/FFMPEGEncoderAV1.h +3dae38db05bc55af525000c28559ff76625e0cddc2ad2c4c2af52eab69e915bc components/FFMPEGEncoderH264.h +3ad5cc5d3284b253a671499efa29c7a46b83a6e6373aad7142c7adcbfae31875 components/FFMPEGEncoderHEVC.h +be992e14086b04e2a7b5503471a6c262c58fda163c45ad6d00f8e425d9f4a820 components/FFMPEGFileDemuxer.h +3fefbe1aa7aaa4c000a97efb2e5ba26dc87ec15d11b07ceb0f43c768a1d589a2 components/FFMPEGFileMuxer.h +297b0ab2291baeb44f30f03ffbf791a5603289ab34f625d60a33150f2be1f7b6 components/FFMPEGVideoDecoder.h +ce5fb69c0546dd08dff0ee680972c5ec5a0b2d9b95e509338358eb2827f30221 components/FRC.h +b63e856b55db7a6853f444a6d4d32cfe006d9e0fb1d9be1b3079557534738591 components/HQScaler.h +c4e168d29e3f7d9318435958c3c784ad4c497159a617bd7a5598a2951b1b15f6 components/MediaSource.h +dae3564f20ef5db959d31dfc74c226453b039f4919ddb65027a183cdae95c171 components/PreAnalysis.h +5a01d71e128cf8bcaa4945fe76d9aad0156c409df3b25616a75645ef15972dc3 components/PreProcessing.h +550900e56c8254bf9c42d1b1d6538a390f7f7d3bf4826ee385b019585cb6cd22 components/SupportedCodecs.h +146e7cd35acc846f6b75b696bbe13a5d790f885bc23d9913e7de66f515da789b components/VQEnhancer.h +08f0162a631833f847d1b4b751e41e28b147a79ed19545b43631222d22e74507 components/VideoCapture.h +b0e173fbfe920a4923f461c1bffc73d1cbc06d54b676647dcd6ec300a0beee65 components/VideoConverter.h +588622f7d945c85c1ed334b497174fd46031e01664338bcf59c4b242920df886 components/VideoDecoderUVD.h +0b43ba9a0bb0d701c117e038d07270907d840ea2d2b13655fec71eda5e96d562 components/VideoEncoderAV1.h +b7827d2c3380a32cef350cf0f180802694f9f93db295f1c8e08238160d17c3d8 components/VideoEncoderHEVC.h +d106ce15028a9ea42e7299cff4b1a8697614a746b96740c9251f9b943bac2edc components/VideoEncoderVCE.h +9bc2d8477d67693b0cd4d433fbc6a5812277190f67be5f188f0206b463f256ec components/VideoStitch.h +3c6e6515a8fec111eb1847e349124a8c66c1e78be2d148bc0b8f85b9d9994e2d components/ZCamLiveStream.h +c9382c911fd3b5e12105c1d2a67eb11e62e9475068825222a285bb1314ac0ddd core/AudioBuffer.h +1ad18d3c813c3f81a836d393bbe86dfc0b7a1d7f04a77a272eb1461ce85ae3a4 core/Buffer.h +ef5721466fe48e1c038151018699519761e6e351e66e76a5de65386924653e20 core/Compute.h +50acf69fb670bc6cf093384d1b698bf9e44cda8dd49e02ef24eacfb86ad8ccae core/ComputeFactory.h +f9b4acb784038aad234ddec490e5f6ad6dacc4b14794eb84f3523d56dc3d6dd2 core/Context.h +76d16ba57d48ae9e457f3d904698a1e7c625007ac8887db9bcecdb90fd36d835 core/CurrentTime.h +f05771305598b534e1efa0dd28825267ea87418c477b198e373f6764d060c666 core/D3D12AMF.h +eaac4d80dd3215638c70f8a4ca01791d26d9df77586f665a772bb51907a8035b core/Data.h +56666589bf6ad20c6286a048da751c2d7722f70c73d962ee3c205a6d709b6aff core/Debug.h +a5e0fa9290cae767a4d135962ee5e30302e65e8d0ca5e016c620e72b759c86c6 core/Dump.h +9ba2ebe93af628bce621f29de493cddab87bf8a030eed504b25fcdc822c61b44 core/Factory.h +304f6d6077e9b0733859d44a5e43a343ae374c8fe72c7d79741c0aca89c21162 core/Interface.h +5ad5165e9de2afcfc2da828e8704c838b1f82e8ead7422fbcfc32c758672e1f3 core/Plane.h +59a4d11ecd150af25ee0b8d1d15f2efdfe5cf1060b8e8ab539f6a46d4bff8bc8 core/Platform.h +0059b787e78cd7c4e57212ccaab56ad7e17e9474a31337da0eac5b2d056a5dd8 core/PropertyStorage.h +871136e46586baa1fc77121935b503aaa16523d006c3e435ce50698605675fad core/PropertyStorageEx.h +a469ff33875484b5762199da757819056ff4a7535bfdcec1f4cf2ffc403b98f2 core/Result.h +7f0c99bbfe156169440031f75b0e37c28c435dd140ecb4e5089d0b1f01912c47 core/Surface.h +ae8c5fee671f02b90d3ccab312cfa6f74424351f070a7dd4ed7aba437fd55c30 core/Trace.h +72841e751595a2a8f62c8ac4a3e630b181411af8e0741d716bbe3379c9bf9a99 core/Variant.h +e26e25e658513f7120bb16d64d753d99205b456d83a1120a0a0d2ffdbf19f8c5 core/Version.h +385b8ad96b59e2d67a6f7a78c6aa660a2f2cdad1bf8c7cadcd83387a3ad4996c core/VulkanAMF.h +AMF_HEADERS +export PKG_CONFIG_PATH="$prefix/lib/pkgconfig" +pkg-config --modversion ffnvcodec vpl +echo "SUCCESS: hardware encoder dependencies installed in $prefix" diff --git a/ci/windows/smoke-ffmpeg-windows.sh b/ci/windows/smoke-ffmpeg-windows.sh new file mode 100755 index 000000000..8db1f12cb --- /dev/null +++ b/ci/windows/smoke-ffmpeg-windows.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +arch=${1:?mingw64 or mingw32} +prefix=${PREFIX:-/c/runner-setup/ffmpeg-7.1.3-$arch} +export PATH="$prefix/bin:/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" +exec > >(tee "/c/runner-setup/logs/ffmpeg-smoke-$arch.log") 2>&1 +"$prefix/bin/ffmpeg.exe" -version +"$prefix/bin/ffmpeg.exe" -hide_banner -hwaccels | grep vulkan +"$prefix/bin/ffmpeg.exe" -hide_banner -encoders > "$prefix/compiled-encoders.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$prefix/compiled-encoders.txt" +done +"$prefix/bin/ffmpeg.exe" -hide_banner -devices > "$prefix/compiled-devices.txt" 2>&1 +grep -q gdigrab "$prefix/compiled-devices.txt" +grep -q dshow "$prefix/compiled-devices.txt" +"$prefix/bin/ffmpeg.exe" -hide_banner -f lavfi -i testsrc2=size=320x240:rate=24 -f lavfi -i sine=frequency=440:sample_rate=48000 -t 2 -c:v libx264 -preset fast -pix_fmt yuv420p -c:a aac -y "$prefix/h264-aac-test.mp4" +"$prefix/bin/ffmpeg.exe" -v error -i "$prefix/h264-aac-test.mp4" -f null - +"$prefix/bin/ffprobe.exe" -v error -show_entries stream=codec_name,width,height,sample_rate "$prefix/h264-aac-test.mp4" +"$prefix/bin/ffmpeg.exe" -v error -i /c/runner-setup/libopenshot-vulkan-mvp/examples/test_video_sync.mp4 -frames:v 3 -f null - +echo "SUCCESS: vendor encoders, screen/camera capture, Vulkan compiled in; H.264/AAC encode and decode $arch" diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index dcb04d74c..c4018f5b8 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -2117,7 +2117,10 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { if (hw_en_av_pix_fmt != AV_PIX_FMT_VULKAN) { video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE; av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0); - av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0); + if (hw_en_av_pix_fmt == AV_PIX_FMT_CUDA) + av_opt_set_int(video_codec_ctx->priv_data, "zerolatency", 1, 0); + else + av_opt_set(video_codec_ctx->priv_data, "tune", "zerolatency", 0); av_opt_set(video_codec_ctx->priv_data, "vprofile", "baseline", AV_OPT_SEARCH_CHILDREN); } break; diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index d9c42a233..a85eaca92 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -91,6 +91,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { }; auto context = QOpenGLContext::currentContext(); if (!context) return unavailable("No current Qt OpenGL context"); + if (context->isOpenGLES()) + return unavailable("Vulkan presentation requires desktop OpenGL; Qt selected OpenGL ES/ANGLE (try QT_OPENGL=desktop)"); if (!frame || !widget) return unavailable("No video frame or preview widget"); if (fallback_delivered) return false; auto authoritative = frame->GetDeferredImage(); diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index a93fab5b6..f076e129f 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -153,6 +153,53 @@ TEST_CASE("Native Vulkan export keeps decoded and composited frames on the GPU", reader.Close(); } } +TEST_CASE("NVENC export accepts CPU and Vulkan frames without changing the requested encoder", "[vulkan][hardware][writer][nvenc]") { + if (!VulkanAvailable()) return; + const auto encoders = openshot::GpuEncoders(); + if (std::find(encoders.begin(), encoders.end(), "h264_nvenc") == encoders.end()) { + WARN("Skipping Vulkan test: no usable NVENC encoder"); + return; + } + QTemporaryDir directory; + REQUIRE(directory.isValid()); + QImage pixels(256, 256, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < pixels.height(); ++y) + for (int x = 0; x < pixels.width(); ++x) + pixels.setPixelColor(x, y, y < 128 ? Qt::red : Qt::blue); + const auto uploaded = openshot::UploadVulkanImage(pixels); + REQUIRE(uploaded); + for (const std::string codec : {"h264_nvenc", "hevc_nvenc"}) { + if (std::find(encoders.begin(), encoders.end(), codec) == encoders.end()) continue; + for (bool gpu : {false, true}) { + INFO("encoder=" << codec << " GPU input=" << gpu); + const auto path = directory.filePath(QString::fromStdString(codec + (gpu ? "-gpu.mp4" : "-cpu.mp4"))).toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true, codec, openshot::Fraction(24, 1), 256, 256, + openshot::Fraction(1, 1), false, false, 2000000); + writer.Open(); + REQUIRE(writer.info.vcodec == codec); + const auto before = openshot::VulkanFrameReadbackCount(); + for (int n = 1; n <= 8; ++n) { + auto frame = std::make_shared(n, 256, 256, "black"); + if (gpu) frame->AddDeferredImage(uploaded); + else frame->AddImage(std::make_shared(pixels)); + writer.WriteFrame(frame); + } + writer.Close(); + if (gpu) CHECK(openshot::VulkanFrameReadbackCount() == before); + openshot::FFmpegReader reader(path); + reader.Open(); + CHECK(reader.info.video_length == 8); + const auto image = reader.GetFrame(4)->GetImage(); + REQUIRE(image); + CHECK(image->pixelColor(128, 64).red() > 220); + CHECK(image->pixelColor(128, 64).blue() < 25); + CHECK(image->pixelColor(128, 192).blue() > 220); + CHECK(image->pixelColor(128, 192).red() < 25); + reader.Close(); + } + } +} TEST_CASE("Vulkan export accepts CPU frames and keeps writers independent", "[vulkan][writer][fallback]") { // Run this with OPENSHOT_RENDER_BACKEND=cpu, an invalid Vulkan device, or // an older FFmpeg too: the requested codec must fall back before the header. diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index e28609e96..6b6fa2187 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -68,6 +68,11 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ int argc = 1; char name[] = "vulkan-presentation"; char* argv[] = {name, nullptr}; +#ifdef _WIN32 + // Match the editor: ANGLE/Direct3D cannot import Vulkan Win32 resources. + if (!qEnvironmentVariableIsSet("QT_OPENGL")) + qputenv("QT_OPENGL", "desktop"); +#endif QApplication app(argc, argv); data.pool = av_hwframe_ctx_alloc(data.device); REQUIRE(data.pool); From 05eb19d7c3380e4c6bbc91c965546c9cd9539ed8 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 00:26:48 -0500 Subject: [PATCH 24/50] Add QSV and AMF to Linux FFmpeg builds --- .gitlab-ci.yml | 2 + ci/linux/check-ffmpeg-hardware.sh | 21 +++ ci/linux/setup-ffmpeg-vulkan.sh | 218 ++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+) create mode 100755 ci/linux/check-ffmpeg-hardware.sh create mode 100755 ci/linux/setup-ffmpeg-vulkan.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2f802b89..b8eb022e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -33,6 +33,7 @@ linux-builder: expire_in: 6 months paths: - build/install-x64/* + - build/ffmpeg-hardware-check/* script: - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot-audio/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder" - if [ ! -f artifacts.zip ]; then @@ -48,6 +49,7 @@ linux-builder: - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT (FFmpeg 7 with Vulkan)"; exit 1; fi - . "$FFMPEG_ROOT/env.sh" - export FFMPEGDIR="$FFMPEG_ROOT" + - bash ci/linux/check-ffmpeg-hardware.sh "$FFMPEG_ROOT" "$CI_PROJECT_DIR/build/ffmpeg-hardware-check" - mkdir -p build; cd build; - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support" - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ diff --git a/ci/linux/check-ffmpeg-hardware.sh b/ci/linux/check-ffmpeg-hardware.sh new file mode 100755 index 000000000..6289d7cd8 --- /dev/null +++ b/ci/linux/check-ffmpeg-hardware.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Compiled support checks are GPU-independent. Real driver/encoding tests are separate. +set -euo pipefail +root=${1:?FFmpeg installation prefix} +log=${2:-$root/hardware-validation} +mkdir -p "$log" +export LD_LIBRARY_PATH="$root/lib:${LD_LIBRARY_PATH:-}" +"$root/bin/ffmpeg" -hide_banner -version > "$log/version.txt" +"$root/bin/ffmpeg" -hide_banner -encoders > "$log/encoders.txt" 2>&1 +"$root/bin/ffmpeg" -hide_banner -hwaccels > "$log/hwaccels.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi h264_vulkan hevc_vulkan; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$log/encoders.txt" || { echo "Missing required encoder: $codec"; exit 1; } +done +ldd "$root/lib/libavcodec.so.61" > "$log/avcodec-dependencies.txt" +if grep -q 'not found' "$log/avcodec-dependencies.txt"; then cat "$log/avcodec-dependencies.txt"; exit 1; fi +grep -q 'libvpl.so.2' "$log/avcodec-dependencies.txt" +"$root/bin/ffmpeg" -v error -f lavfi -i testsrc2=size=320x240:rate=24 -f lavfi -i sine=sample_rate=48000 -t 2 -c:v libx264 -threads 2 -c:a aac -y "$log/h264-aac.mp4" +"$root/bin/ffmpeg" -v error -i "$log/h264-aac.mp4" -f null - +"$root/bin/ffmpeg" -v error -f lavfi -i testsrc2=size=128x96:rate=4 -frames:v 4 -c:v libaom-av1 -cpu-used 8 -threads 2 -y "$log/av1.webm" +"$root/bin/ffmpeg" -v error -c:v libdav1d -i "$log/av1.webm" -f null - +echo 'SUCCESS: NVENC, QSV, AMF, VA-API and Vulkan encoders compiled; dispatcher linked; H.264/AAC and AV1 software smoke tests passed' diff --git a/ci/linux/setup-ffmpeg-vulkan.sh b/ci/linux/setup-ffmpeg-vulkan.sh new file mode 100755 index 000000000..88012992b --- /dev/null +++ b/ci/linux/setup-ffmpeg-vulkan.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# Install FFmpeg with Vulkan (plus the codecs the focal AppImage ships) into +# /opt, without touching the system FFmpeg used by the CPU-only builds. +# Ubuntu 20.04 (focal). Run as a sudo-capable user. Safe to re-run. +# Not enabled: libwavpack (removed from FFmpeg 5+, native decoder used), +# chromaprint (focal's library links FFmpeg 4.2, loading two FFmpegs at once). +set -euo pipefail + +FFMPEG_VERSION=${FFMPEG_VERSION:-7.1.3} +VULKAN_HEADERS_TAG=${VULKAN_HEADERS_TAG:-v1.4.347} +AOM_TAG=${AOM_TAG:-v3.15.2} # focal has 1.0, FFmpeg 7 needs 2.0+ +DAV1D_TAG=${DAV1D_TAG:-1.4.3} # focal has no libdav1d-dev +NV_CODEC_TAG=${NV_CODEC_TAG:-n12.1.14.0} # NVIDIA driver 530+ for NVENC +PREFIX=${PREFIX:-/opt/ffmpeg-${FFMPEG_VERSION}} +VULKAN_PREFIX=${VULKAN_PREFIX:-/opt/vulkan-headers} +SRC=${SRC:-/opt/src} +JOBS=${JOBS:-$(nproc)} +SKIP_APT=${SKIP_APT:-0} + +export DEBIAN_FRONTEND=noninteractive +if [ "$SKIP_APT" != 1 ]; then +sudo apt-get update +sudo apt-get install -y --no-install-recommends \ + build-essential pkg-config nasm yasm git curl xz-utils cmake \ + glslang-tools libvulkan-dev \ + libx264-dev libx265-dev libvpx-dev meson ninja-build \ + libmp3lame-dev libopus-dev libvorbis-dev libtheora-dev libspeex-dev \ + libtwolame-dev libshine-dev libwavpack-dev libgsm1-dev libcodec2-dev \ + libxvidcore-dev libwebp-dev libopenjp2-7-dev libsnappy-dev libsoxr-dev \ + libass-dev libfreetype6-dev libfontconfig1-dev libfribidi-dev libxml2-dev \ + libbluray-dev libbs2b-dev libcaca-dev libcdio-paranoia-dev flite1-dev \ + libgme-dev libmysofa-dev libopenmpt-dev librsvg2-dev librubberband-dev \ + libssh-dev libgnutls28-dev libvidstab-dev libzmq3-dev libzvbi-dev \ + lv2-dev liblilv-dev ladspa-sdk frei0r-plugins-dev \ + libpulse-dev libjack-jackd2-dev libopenal-dev libsdl2-dev \ + ocl-icd-opencl-dev opencl-headers libgl-dev \ + libdc1394-22-dev libiec61883-dev libavc1394-dev libraw1394-dev \ + libdrm-dev libva-dev libvdpau-dev +fi + +sudo mkdir -p "$SRC" && sudo chown "$(id -u):$(id -g)" "$SRC" +cd "$SRC" + +# Vulkan headers: focal has 1.2.131, FFmpeg 7 needs 1.3.277+. Header-only. +if [ ! -f "$VULKAN_PREFIX/include/vulkan/vulkan_core.h" ]; then + rm -rf Vulkan-Headers + git clone --depth 1 --branch "$VULKAN_HEADERS_TAG" https://github.com/KhronosGroup/Vulkan-Headers.git + # Copy directly: its CMakeLists needs CMake 3.22, focal has 3.16 + sudo mkdir -p "$VULKAN_PREFIX" + sudo cp -r Vulkan-Headers/include "$VULKAN_PREFIX/" +fi + +# NVENC/CUDA headers (header-only, loaded at runtime from the driver) +if [ ! -f "$PREFIX/lib/pkgconfig/ffnvcodec.pc" ]; then + rm -rf nv-codec-headers + git clone --depth 1 --branch "$NV_CODEC_TAG" https://github.com/FFmpeg/nv-codec-headers.git + sudo make -C nv-codec-headers install PREFIX="$PREFIX" +fi + +# Intel oneVPL dispatcher (supports both oneVPL and legacy Media SDK runtimes). +# The dispatcher is bundled; GPU runtimes/drivers remain supplied by the host. +if [ ! -f "$PREFIX/lib/pkgconfig/vpl.pc" ]; then + archive=libvpl-2.10.2.tar.gz + if [ ! -f "$archive" ]; then + curl -fsSL --retry 2 https://github.com/intel/libvpl/archive/refs/tags/v2.10.2.tar.gz -o "$archive" + fi + printf '%s %s\n' ad956ea7ecf14614325f59dfb44cc5ba08e2fcac373342d61c7db152ac651253 "$archive" | sha256sum -c - + mkdir -p libvpl-2.10.2 + tar xzf "$archive" --strip-components=1 -C libvpl-2.10.2 + cmake -S libvpl-2.10.2 -B libvpl-2.10.2/build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTS=OFF -DBUILD_TOOLS=OFF -DBUILD_EXAMPLES=OFF -DINSTALL_EXAMPLE_CODE=OFF \ + -DBUILD_DISPATCHER_ONEVPL_EXPERIMENTAL=OFF + cmake --build libvpl-2.10.2/build --parallel "$JOBS" + sudo cmake --install libvpl-2.10.2/build +fi + +# AMD AMF headers, pinned to the 1.4.35 release commit and verified individually. +# FFmpeg loads the AMF runtime from the installed AMD driver at export time. +amf="$PREFIX/include/AMF" +sudo mkdir -p "$amf" +while read -r digest header; do + if ! printf '%s %s\n' "$digest" "$amf/$header" | sha256sum -c - >/dev/null 2>&1; then + mkdir -p "amf-headers/$(dirname "$header")" + curl -fsSL --retry 2 "https://raw.githubusercontent.com/GPUOpen-LibrariesAndSDKs/AMF/ba07d1b3c7c7ee0d846125cffc1d6d78a020c45a/amf/public/include/$header" -o "amf-headers/$header" + printf '%s %s\n' "$digest" "amf-headers/$header" | sha256sum -c - + sudo mkdir -p "$amf/$(dirname "$header")" + sudo cp "amf-headers/$header" "$amf/$header" + fi +done <<'AMF_HEADERS' +70afd5279be90e656c2ec270229c5a702a11435a670052370e96b270e6b7640f components/AMFXInput.h +27770ae77d2f7a16eca9ab1f8b9ce87cf0c0350425b950bdd39189c2c91a519a components/Ambisonic2SRenderer.h +05a9ebc32217e69b6c4659f4744dbda9e5a2792f3e56bc1bcf61c15dcfd46cab components/AudioCapture.h +bbe91a1e744685a36269d3b6af9cd36d9716ee748c8d0ed95255a6a42b46892f components/Capture.h +a3a89490d44adaffc2fa2cccc14313f8f65e5af7a6431d89d85bd80e36c628a1 components/ChromaKey.h +2f6887d14516ebdcf034d59e39235f5d9974533ddb0b04d685c0c3b6d397d3d6 components/ColorSpace.h +f3f18680d55a65fdf5e0351d3fde1dcd9bdd2d6caec268c8c915c4227ffa1edc components/Component.h +2479eb2a6174a1b63552c509abac1b701eddcab5a5eac5a7f4d9f40274d588b4 components/ComponentCaps.h +7c22337ff313320bd7fe20ef0fb00756735012367a5daa7c621725030f578863 components/CursorCapture.h +693c8e491f90eaffd226c1f6b2da37d3a5ef1d3c610e554647f7d0b699832727 components/DisplayCapture.h +9dde7e850c8bdecda82b0cee1b331294f6cc91b32b1d1504344a023cd4dda8af components/FFMPEGAudioConverter.h +92f50d789bc18d5ebd143b2925e124594a89110eef7efaa8189eb063bf2f3aa8 components/FFMPEGAudioDecoder.h +2493067de9b3f540ffaae8a6466e57ef75af4772f037a81785b11ee9511029f5 components/FFMPEGAudioEncoder.h +87a9ae9480c3af6210b1282d1768e36673fcc1ef4970afa6edaa74481cf0831b components/FFMPEGComponents.h +5d64b7d2d50ca17ba601885dcdb51bb16ef4dccf88323437967e8524f26194bc components/FFMPEGEncoderAV1.h +3dae38db05bc55af525000c28559ff76625e0cddc2ad2c4c2af52eab69e915bc components/FFMPEGEncoderH264.h +3ad5cc5d3284b253a671499efa29c7a46b83a6e6373aad7142c7adcbfae31875 components/FFMPEGEncoderHEVC.h +be992e14086b04e2a7b5503471a6c262c58fda163c45ad6d00f8e425d9f4a820 components/FFMPEGFileDemuxer.h +3fefbe1aa7aaa4c000a97efb2e5ba26dc87ec15d11b07ceb0f43c768a1d589a2 components/FFMPEGFileMuxer.h +297b0ab2291baeb44f30f03ffbf791a5603289ab34f625d60a33150f2be1f7b6 components/FFMPEGVideoDecoder.h +ce5fb69c0546dd08dff0ee680972c5ec5a0b2d9b95e509338358eb2827f30221 components/FRC.h +b63e856b55db7a6853f444a6d4d32cfe006d9e0fb1d9be1b3079557534738591 components/HQScaler.h +c4e168d29e3f7d9318435958c3c784ad4c497159a617bd7a5598a2951b1b15f6 components/MediaSource.h +dae3564f20ef5db959d31dfc74c226453b039f4919ddb65027a183cdae95c171 components/PreAnalysis.h +5a01d71e128cf8bcaa4945fe76d9aad0156c409df3b25616a75645ef15972dc3 components/PreProcessing.h +550900e56c8254bf9c42d1b1d6538a390f7f7d3bf4826ee385b019585cb6cd22 components/SupportedCodecs.h +146e7cd35acc846f6b75b696bbe13a5d790f885bc23d9913e7de66f515da789b components/VQEnhancer.h +08f0162a631833f847d1b4b751e41e28b147a79ed19545b43631222d22e74507 components/VideoCapture.h +b0e173fbfe920a4923f461c1bffc73d1cbc06d54b676647dcd6ec300a0beee65 components/VideoConverter.h +588622f7d945c85c1ed334b497174fd46031e01664338bcf59c4b242920df886 components/VideoDecoderUVD.h +0b43ba9a0bb0d701c117e038d07270907d840ea2d2b13655fec71eda5e96d562 components/VideoEncoderAV1.h +b7827d2c3380a32cef350cf0f180802694f9f93db295f1c8e08238160d17c3d8 components/VideoEncoderHEVC.h +d106ce15028a9ea42e7299cff4b1a8697614a746b96740c9251f9b943bac2edc components/VideoEncoderVCE.h +9bc2d8477d67693b0cd4d433fbc6a5812277190f67be5f188f0206b463f256ec components/VideoStitch.h +3c6e6515a8fec111eb1847e349124a8c66c1e78be2d148bc0b8f85b9d9994e2d components/ZCamLiveStream.h +c9382c911fd3b5e12105c1d2a67eb11e62e9475068825222a285bb1314ac0ddd core/AudioBuffer.h +1ad18d3c813c3f81a836d393bbe86dfc0b7a1d7f04a77a272eb1461ce85ae3a4 core/Buffer.h +ef5721466fe48e1c038151018699519761e6e351e66e76a5de65386924653e20 core/Compute.h +50acf69fb670bc6cf093384d1b698bf9e44cda8dd49e02ef24eacfb86ad8ccae core/ComputeFactory.h +f9b4acb784038aad234ddec490e5f6ad6dacc4b14794eb84f3523d56dc3d6dd2 core/Context.h +76d16ba57d48ae9e457f3d904698a1e7c625007ac8887db9bcecdb90fd36d835 core/CurrentTime.h +f05771305598b534e1efa0dd28825267ea87418c477b198e373f6764d060c666 core/D3D12AMF.h +eaac4d80dd3215638c70f8a4ca01791d26d9df77586f665a772bb51907a8035b core/Data.h +56666589bf6ad20c6286a048da751c2d7722f70c73d962ee3c205a6d709b6aff core/Debug.h +a5e0fa9290cae767a4d135962ee5e30302e65e8d0ca5e016c620e72b759c86c6 core/Dump.h +9ba2ebe93af628bce621f29de493cddab87bf8a030eed504b25fcdc822c61b44 core/Factory.h +304f6d6077e9b0733859d44a5e43a343ae374c8fe72c7d79741c0aca89c21162 core/Interface.h +5ad5165e9de2afcfc2da828e8704c838b1f82e8ead7422fbcfc32c758672e1f3 core/Plane.h +59a4d11ecd150af25ee0b8d1d15f2efdfe5cf1060b8e8ab539f6a46d4bff8bc8 core/Platform.h +0059b787e78cd7c4e57212ccaab56ad7e17e9474a31337da0eac5b2d056a5dd8 core/PropertyStorage.h +871136e46586baa1fc77121935b503aaa16523d006c3e435ce50698605675fad core/PropertyStorageEx.h +a469ff33875484b5762199da757819056ff4a7535bfdcec1f4cf2ffc403b98f2 core/Result.h +7f0c99bbfe156169440031f75b0e37c28c435dd140ecb4e5089d0b1f01912c47 core/Surface.h +ae8c5fee671f02b90d3ccab312cfa6f74424351f070a7dd4ed7aba437fd55c30 core/Trace.h +72841e751595a2a8f62c8ac4a3e630b181411af8e0741d716bbe3379c9bf9a99 core/Variant.h +e26e25e658513f7120bb16d64d753d99205b456d83a1120a0a0d2ffdbf19f8c5 core/Version.h +385b8ad96b59e2d67a6f7a78c6aa660a2f2cdad1bf8c7cadcd83387a3ad4996c core/VulkanAMF.h +AMF_HEADERS + +# dav1d (fast AV1 decoder), built into the FFmpeg prefix +if [ ! -f "$PREFIX/lib/pkgconfig/dav1d.pc" ]; then + rm -rf dav1d + git clone --depth 1 --branch "$DAV1D_TAG" https://code.videolan.org/videolan/dav1d.git + meson setup dav1d/build dav1d --prefix="$PREFIX" --libdir=lib --buildtype=release \ + -Denable_tools=false -Denable_tests=false + ninja -C dav1d/build + sudo ninja -C dav1d/build install +fi + +# libaom (AV1 encoding), built into the FFmpeg prefix +if [ ! -f "$PREFIX/lib/pkgconfig/aom.pc" ]; then + rm -rf aom + git clone --depth 1 --branch "$AOM_TAG" https://aomedia.googlesource.com/aom + cmake -S aom -B aom/build-out -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=1 \ + -DENABLE_TESTS=0 -DENABLE_EXAMPLES=0 -DENABLE_DOCS=0 -DENABLE_TOOLS=0 + cmake --build aom/build-out --parallel "$JOBS" + sudo cmake --install aom/build-out +fi + +if [ ! -d "ffmpeg-$FFMPEG_VERSION" ]; then + # GitHub mirror: ffmpeg.org is not always reachable from the runners + archive="FFmpeg-n$FFMPEG_VERSION.tar.gz" + curl -fsSL --retry 2 "https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n$FFMPEG_VERSION.tar.gz" -o "$archive" + printf '%s %s\n' "${FFMPEG_SHA256:-e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb}" "$archive" | sha256sum -c - + tar xzf "$archive" + mv "FFmpeg-n$FFMPEG_VERSION" "ffmpeg-$FFMPEG_VERSION" +fi +cd "ffmpeg-$FFMPEG_VERSION" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +./configure --prefix="$PREFIX" --enable-gpl --enable-version3 \ + --enable-shared --disable-static --disable-doc \ + --extra-cflags="-I$VULKAN_PREFIX/include -I$PREFIX/include" \ + --extra-ldflags="-L$PREFIX/lib" \ + --enable-vulkan --enable-ffnvcodec --enable-nvenc --enable-nvdec --enable-cuvid \ + --enable-libvpl --enable-amf \ + --enable-vaapi --enable-vdpau --enable-libdrm --enable-opencl --enable-opengl \ + --enable-libx264 --enable-libx265 --enable-libvpx --enable-libaom --enable-libdav1d \ + --enable-libmp3lame --enable-libopus --enable-libvorbis --enable-libtheora \ + --enable-libspeex --enable-libtwolame --enable-libshine \ + --enable-libgsm --enable-libcodec2 --enable-libxvid --enable-libwebp \ + --enable-libopenjpeg --enable-libsnappy --enable-libsoxr \ + --enable-libass --enable-libfreetype --enable-libfontconfig --enable-libfribidi \ + --enable-libxml2 --enable-libbluray --enable-libbs2b --enable-libcaca \ + --enable-libcdio --enable-libflite --enable-libgme --enable-libmysofa \ + --enable-libopenmpt --enable-librsvg --enable-librubberband --enable-libssh \ + --enable-gnutls --enable-libvidstab --enable-libzmq --enable-libzvbi \ + --enable-lv2 --enable-ladspa --enable-frei0r \ + --enable-libpulse --enable-libjack --enable-openal --enable-sdl2 \ + --enable-libdc1394 --enable-libiec61883 +make -j"$JOBS" +sudo make install + +# Build env for GPU jobs: source this before cmake. +sudo tee "$PREFIX/env.sh" >/dev/null < "$PREFIX/compiled-encoders.txt" 2>&1 +for codec in h264_nvenc hevc_nvenc h264_qsv hevc_qsv h264_amf hevc_amf h264_vaapi hevc_vaapi h264_vulkan hevc_vulkan; do + grep -Eq "[[:space:]]$codec[[:space:]]" "$PREFIX/compiled-encoders.txt" || { echo "Required encoder missing: $codec"; exit 1; } +done +echo "Done: source $PREFIX/env.sh" From 9ddcede854be1fd1dea9db9186ce3d64dd871a0d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 13:24:05 -0600 Subject: [PATCH 25/50] Diagnose GPU presentation and allow CPU display fallback --- src/Qt/VulkanOpenGLPresenter.cpp | 54 ++++++++++++++++++++++++++++++-- src/VulkanCompositor.cpp | 22 +++++++++++-- tests/VulkanPresentation.cpp | 17 ++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index a85eaca92..972221ab5 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -4,6 +4,7 @@ #include "VideoRenderer.h" #include "../VulkanFrameImage.h" #include "../VulkanCompositor.h" +#include "../Logger.h" #include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include "ExternalOpenGL.h" #ifdef _WIN32 @@ -49,6 +51,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { uint64_t native_draws = 0; uint64_t fallback_frames = 0; std::string last_failure; + QPointer logged_context; + bool identity_logged = false; bool unavailable(const std::string& reason) { last_failure = reason; return false; } explicit Impl(QWidget* value) : widget(value) { value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); @@ -81,6 +85,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { } } bool draw(int x, int y, int width, int height) { + if (qEnvironmentVariable("OPENSHOT_PRESENTATION_BACKEND") == "cpu") + return unavailable("OPENSHOT_PRESENTATION_BACKEND=cpu (Vulkan processing remains enabled)"); auto lastStage = std::chrono::steady_clock::now(); const bool trace = qEnvironmentVariableIsSet("OPENSHOT_TRACE_PRESENTATION"); auto stage = [&](const char* name) { @@ -91,6 +97,27 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { }; auto context = QOpenGLContext::currentContext(); if (!context) return unavailable("No current Qt OpenGL context"); + if (logged_context != context) { + auto text = [&](GLenum name) { + const auto* value = context->functions()->glGetString(name); + return value ? reinterpret_cast(value) : "unknown"; + }; +#ifdef _WIN32 + const bool memory = context->hasExtension("GL_EXT_memory_object_win32"); + const bool semaphore = context->hasExtension("GL_EXT_semaphore_win32"); +#else + const bool memory = context->hasExtension("GL_EXT_memory_object_fd"); + const bool semaphore = context->hasExtension("GL_EXT_semaphore_fd"); +#endif + Logger::Instance()->Log(std::string("GPU presentation Qt context: vendor=") + text(GL_VENDOR) + + "; renderer=" + text(GL_RENDERER) + "; version=" + text(GL_VERSION) + + "; OpenGL ES=" + (context->isOpenGLES() ? "yes" : "no") + + "; external memory=" + (memory ? "yes" : "no") + + "; external semaphore=" + (semaphore ? "yes" : "no"), Logger::LevelInfo); + logged_context = context; + identity_logged = false; + } + stage("begin draw"); if (context->isOpenGLES()) return unavailable("Vulkan presentation requires desktop OpenGL; Qt selected OpenGL ES/ANGLE (try QT_OPENGL=desktop)"); if (!frame || !widget) return unavailable("No video frame or preview widget"); @@ -101,6 +128,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (!surface) return unavailable("Frame cannot be presented as a Vulkan RGBA surface"); auto inputContext = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); if (inputContext->sw_format != AV_PIX_FMT_RGBA) { + stage("begin RGBA conversion"); surface = std::dynamic_pointer_cast(RenderVulkanImage( surface, nullptr, surface->GetWidth(), surface->GetHeight(), QTransform(), 1.0, Qt::black)); if (!surface) return unavailable("Frame cannot be presented as a Vulkan RGBA surface"); @@ -116,6 +144,16 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { // Multi-plane decode surfaces must first go through the RGBA compositor. if (fc->sw_format != AV_PIX_FMT_RGBA || vfc->format[0] != VK_FORMAT_R8G8B8A8_UNORM || image->img[1]) return unavailable("Unsupported Vulkan image format for OpenGL sharing"); + VkPhysicalDeviceIDProperties identity{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity}; + vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); + const bool logIdentity = !identity_logged; + if (logIdentity) { + Logger::Instance()->Log(std::string("GPU presentation Vulkan device: ") + properties.properties.deviceName + + "; UUID=" + QByteArray(reinterpret_cast(identity.deviceUUID), VK_UUID_SIZE).toHex().toStdString(), + Logger::LevelInfo); + identity_logged = true; + } #ifdef _WIN32 if (!context->hasExtension("GL_EXT_memory_object_win32") || !context->hasExtension("GL_EXT_semaphore_win32")) #else @@ -125,15 +163,15 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { auto gl = context->functions(); auto extra = context->extraFunctions(); auto getUuid = glFunction(context, "glGetUnsignedBytei_vEXT"); - VkPhysicalDeviceIDProperties identity{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES}; - VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity}; - vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); GLint deviceCount = 0; gl->glGetIntegerv(GL_NUM_DEVICE_UUIDS_EXT, &deviceCount); bool sameDevice = false; for (int i = 0; i < deviceCount; ++i) { GLubyte uuid[GL_UUID_SIZE_EXT]; getUuid(GL_DEVICE_UUID_EXT, i, uuid); + if (logIdentity) + Logger::Instance()->Log("GPU presentation OpenGL device UUID=" + + QByteArray(reinterpret_cast(uuid), GL_UUID_SIZE_EXT).toHex().toStdString(), Logger::LevelInfo); sameDevice = sameDevice || std::memcmp(uuid, identity.deviceUUID, VK_UUID_SIZE) == 0; } if (!sameDevice) { @@ -362,6 +400,16 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { gl->glDisable(GL_BLEND); gl->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); gl->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + if (trace) { + const auto error = gl->glGetError(); + GLint framebuffer = 0, activeProgram = 0, boundTexture = 0; + gl->glGetIntegerv(GL_FRAMEBUFFER_BINDING, &framebuffer); + gl->glGetIntegerv(GL_CURRENT_PROGRAM, &activeProgram); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &boundTexture); + qWarning() << "GPU presentation draw state: error=" << error + << "framebuffer=" << framebuffer << "program=" << activeProgram + << "texture=" << boundTexture << "vertex=" << vertex; + } signalSemaphore(semaphores[1], 0, nullptr, 1, &texture, &layout); gl->glFlush(); stage("GL draw flush"); diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 5785cc897..b51005b9b 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -12,10 +12,12 @@ #include #include #include +#include #include extern "C" { #include #include +#include } // SPIR-V generated from src/vulkan/*.comp at build time. #include "affine.comp.h" @@ -26,7 +28,11 @@ void VkCheck(VkResult r, const char* operation) { if(r != VK_SUCCESS) throw std::runtime_error(std::string(operation)+": Vulkan error "+std::to_string(r)); } void AvCheck(int r, const char* operation) { - if(r < 0) throw std::runtime_error(std::string(operation)+": FFmpeg error "+std::to_string(r)); + if(r < 0) { + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(r, error, sizeof(error)); + throw std::runtime_error(std::string(operation)+": FFmpeg error "+std::to_string(r)+" ("+error+")"); + } } struct FrameDelete { void operator()(AVFrame* f) const { av_frame_free(&f); } }; using FramePtr=std::unique_ptr; @@ -232,8 +238,20 @@ class Pipeline { #else AVBufferRef* pool=OutputPool(device_ref,width,height,encode_nv12); const int allocated=av_hwframe_get_buffer(pool,output.get(),0); + std::string operation="Allocate compositor output"; + if(allocated<0){ + const auto* fc=reinterpret_cast(pool->data); + const auto* vc=reinterpret_cast(fc->hwctx); + std::ostringstream detail; + detail << operation << " [" << width << 'x' << height + << ", format=" << (encode_nv12?"NV12":"RGBA") + << ", usage=0x" << std::hex << vc->usage + << ", image_flags=0x" << vc->img_flags << std::dec + << ", tiling=" << vc->tiling << ']'; + operation=detail.str(); + } av_buffer_unref(&pool); - AvCheck(allocated,"Allocate compositor output"); + AvCheck(allocated,operation.c_str()); #endif } AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 6b6fa2187..c34c03d9b 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -12,6 +12,7 @@ #include #include #include +#include extern "C" { #include } @@ -20,6 +21,18 @@ class Preview : public QOpenGLWidget { public: openshot::VulkanOpenGLPresenter presenter{reinterpret_cast(this)}; bool nativeDraw = false; + void initializeGL() override { + auto* current = context(); + typedef const GLubyte* (APIENTRYP GetString)(GLenum); + auto getString = reinterpret_cast(current->getProcAddress("glGetString")); + qWarning() << "Presentation test Qt context:" << qVersion() << current->format() + << "OpenGL ES:" << current->isOpenGLES() + << "glCreateProgram:" << bool(current->getProcAddress("glCreateProgram")); + if (getString) + qWarning() << "Presentation test GL:" << reinterpret_cast(getString(GL_VENDOR)) + << reinterpret_cast(getString(GL_RENDERER)) + << reinterpret_cast(getString(GL_VERSION)); + } void paintGL() override { QPainter painter(this); painter.fillRect(rect(), Qt::blue); @@ -122,6 +135,10 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ // a race where Vulkan consumes its own release signal before GL waits. for (int i = 0; i < 64; ++i) { image = widget.grabFramebuffer(); + if (i == 0 && qEnvironmentVariableIsSet("OPENSHOT_PRESENTATION_CAPTURE")) + image.save(qEnvironmentVariable("OPENSHOT_PRESENTATION_CAPTURE")); + INFO("frame=" << i << " top=" << image.pixelColor(image.width()/2, image.height()/4).name().toStdString() + << " bottom=" << image.pixelColor(image.width()/2, 3*image.height()/4).name().toStdString()); REQUIRE(widget.nativeDraw); REQUIRE(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); REQUIRE(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); From 055c66ff9fedafab9e564df104b72462945409ec Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 13:58:21 -0600 Subject: [PATCH 26/50] Present Vulkan frames through a presenter-owned shared image NVIDIA's Windows driver returns zeros when FFmpeg or compositor frames are read through an exported alias of their memory, from OpenGL or Vulkan, so native preview drew black. Copy each frame on the GPU into an exportable image owned by the presenter, which OpenGL imports once per size, device, and context, along with the semaphores, shader, and vertex state. The frame no longer changes queue ownership, which removes the invalid external acquire on concurrent FFmpeg images. The previous frame's fence is waited at the next draw instead of finishing GL every frame. --- src/Qt/VulkanOpenGLPresenter.cpp | 613 +++++++++++++++++++------------ tests/VulkanPresentation.cpp | 17 + 2 files changed, 387 insertions(+), 243 deletions(-) diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 972221ab5..b47be21df 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -42,7 +42,51 @@ template T glFunction(QOpenGLContext* context, const char* name) { if (!function) throw std::runtime_error(name); return function; } +#ifdef _WIN32 +constexpr auto kMemoryHandle = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT; +constexpr auto kSemaphoreHandle = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; +constexpr GLenum kGLHandle = GL_HANDLE_TYPE_OPAQUE_WIN32_EXT; +constexpr const char* kMemoryExtension = "GL_EXT_memory_object_win32"; +constexpr const char* kSemaphoreExtension = "GL_EXT_semaphore_win32"; +using ImportMemory = PFNGLIMPORTMEMORYWIN32HANDLEEXTPROC; +using ImportSemaphore = PFNGLIMPORTSEMAPHOREWIN32HANDLEEXTPROC; +#else +constexpr auto kMemoryHandle = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; +constexpr auto kSemaphoreHandle = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; +constexpr GLenum kGLHandle = GL_HANDLE_TYPE_OPAQUE_FD_EXT; +constexpr const char* kMemoryExtension = "GL_EXT_memory_object_fd"; +constexpr const char* kSemaphoreExtension = "GL_EXT_semaphore_fd"; +using ImportMemory = PFNGLIMPORTMEMORYFDEXTPROC; +using ImportSemaphore = PFNGLIMPORTSEMAPHOREFDEXTPROC; +#endif +constexpr VkImageSubresourceRange kColor = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; } +// OpenGL samples a presenter-owned Vulkan image rather than the frame itself. +// NVIDIA's Windows driver returns zeros when FFmpeg or compositor frames are +// read through an exported alias of their memory (even from Vulkan), while a +// GPU copy into an image allocated for export is visible. The shared image and +// its OpenGL import persist until the frame size, device, or context changes. +struct SharedTarget { + AVBufferRef* device_ref = nullptr; + VkDevice device = VK_NULL_HANDLE; + QPointer context; + int width = 0, height = 0; + uint32_t family = 0; + VkQueue queue = VK_NULL_HANDLE; + VkImage image = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + // Separate directions: reusing one binary semaphore lets Vulkan's + // acquire consume its own release signal before GL can wait on it. + VkSemaphore handoff[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkCommandPool pool = VK_NULL_HANDLE; + VkCommandBuffer commands[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkFence fence = VK_NULL_HANDLE; + bool pending = false; + GLuint gl_memory = 0, texture = 0, gl_semaphores[2] = {0, 0}, buffer = 0, vertex_array = 0; + std::unique_ptr program; + PFNGLWAITSEMAPHOREEXTPROC wait_semaphore = nullptr; + PFNGLSIGNALSEMAPHOREEXTPROC signal_semaphore = nullptr; +}; struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { QPointer widget; std::shared_ptr frame; @@ -53,11 +97,13 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { std::string last_failure; QPointer logged_context; bool identity_logged = false; + SharedTarget target; bool unavailable(const std::string& reason) { last_failure = reason; return false; } explicit Impl(QWidget* value) : widget(value) { value->setProperty("openshot_gpu_presenter", QVariant::fromValue(static_cast(this))); } ~Impl() override { + releaseTarget(); if (widget) widget->setProperty("openshot_gpu_presenter", QVariant()); } void PresentFrame(const std::shared_ptr& value) override { @@ -84,6 +130,210 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); } } + void waitPending() { + if (!target.pending) return; + target.pending = false; + check(vkWaitForFences(target.device, 1, &target.fence, VK_TRUE, UINT64_MAX), "Wait for presentation transfer"); + check(vkResetFences(target.device, 1, &target.fence), "Reset presentation fence"); + } + // OpenGL objects are deleted only while their context is current; otherwise + // they are released with that context. + void releaseTarget() { + if (target.device) { + try { waitPending(); } catch (const std::exception&) { vkDeviceWaitIdle(target.device); } + } + auto* context = QOpenGLContext::currentContext(); + if (target.context && context == target.context) { + auto gl = context->functions(); + if (target.vertex_array) context->extraFunctions()->glDeleteVertexArrays(1, &target.vertex_array); + if (target.buffer) gl->glDeleteBuffers(1, &target.buffer); + if (target.texture) gl->glDeleteTextures(1, &target.texture); + if (target.gl_memory) + reinterpret_cast(context->getProcAddress("glDeleteMemoryObjectsEXT"))(1, &target.gl_memory); + if (target.gl_semaphores[0] || target.gl_semaphores[1]) + reinterpret_cast(context->getProcAddress("glDeleteSemaphoresEXT"))(2, target.gl_semaphores); + } + target.program.reset(); // Qt defers deletion until the context is current. + if (target.device) { + if (target.fence) vkDestroyFence(target.device, target.fence, nullptr); + if (target.pool) vkDestroyCommandPool(target.device, target.pool, nullptr); + for (auto semaphore : target.handoff) + if (semaphore) vkDestroySemaphore(target.device, semaphore, nullptr); + if (target.image) vkDestroyImage(target.device, target.image, nullptr); + if (target.memory) vkFreeMemory(target.device, target.memory, nullptr); + } + av_buffer_unref(&target.device_ref); + target = SharedTarget(); + } + void createTarget(QOpenGLContext* context, const AVHWFramesContext* fc, int width, int height) { + auto vk = static_cast(fc->device_ctx->hwctx); + target.device_ref = av_buffer_ref(fc->device_ref); + if (!target.device_ref) throw std::bad_alloc(); + target.device = vk->act_dev; + target.context = context; + target.width = width; + target.height = height; + target.family = vk->queue_family_comp_index; + vkGetDeviceQueue(target.device, target.family, 0, &target.queue); + auto gl = context->functions(); + auto extra = context->extraFunctions(); + auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); + auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); + auto textureMemory = glFunction(context, "glTexStorageMem2DEXT"); + auto createSemaphores = glFunction(context, "glGenSemaphoresEXT"); + glFunction(context, "glDeleteMemoryObjectsEXT"); + glFunction(context, "glDeleteSemaphoresEXT"); + target.wait_semaphore = glFunction(context, "glWaitSemaphoreEXT"); + target.signal_semaphore = glFunction(context, "glSignalSemaphoreEXT"); +#ifdef _WIN32 + auto importMemory = glFunction(context, "glImportMemoryWin32HandleEXT"); + auto importSemaphore = glFunction(context, "glImportSemaphoreWin32HandleEXT"); + auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetMemoryWin32HandleKHR")); + auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetSemaphoreWin32HandleKHR")); +#else + auto importMemory = glFunction(context, "glImportMemoryFdEXT"); + auto importSemaphore = glFunction(context, "glImportSemaphoreFdEXT"); + auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetMemoryFdKHR")); + auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(target.device, "vkGetSemaphoreFdKHR")); +#endif + if (!getMemoryHandle || !getSemaphoreHandle) + throw std::runtime_error("Vulkan device lacks external memory/semaphore export"); + + VkExternalMemoryImageCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO}; + external.handleTypes = kMemoryHandle; + VkImageCreateInfo create{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, &external}; + create.imageType = VK_IMAGE_TYPE_2D; + create.format = VK_FORMAT_R8G8B8A8_UNORM; + create.extent = {static_cast(width), static_cast(height), 1}; + create.mipLevels = create.arrayLayers = 1; + create.samples = VK_SAMPLE_COUNT_1_BIT; + create.tiling = VK_IMAGE_TILING_OPTIMAL; + create.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + check(vkCreateImage(target.device, &create, nullptr, &target.image), "Create presentation image"); + VkMemoryDedicatedRequirements dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; + VkMemoryRequirements2 requirements{VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated}; + VkImageMemoryRequirementsInfo2 requirementInfo{VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2}; + requirementInfo.image = target.image; + vkGetImageMemoryRequirements2(target.device, &requirementInfo, &requirements); + VkPhysicalDeviceMemoryProperties memoryProperties; + vkGetPhysicalDeviceMemoryProperties(vk->phys_dev, &memoryProperties); + uint32_t memoryType = UINT32_MAX; + for (uint32_t i = 0; i < memoryProperties.memoryTypeCount && memoryType == UINT32_MAX; ++i) + if ((requirements.memoryRequirements.memoryTypeBits & (1u << i)) && + (memoryProperties.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) + memoryType = i; + if (memoryType == UINT32_MAX) throw std::runtime_error("No device-local presentation memory"); + const bool useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; + VkMemoryDedicatedAllocateInfo dedicatedInfo{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO}; + dedicatedInfo.image = target.image; + VkExportMemoryAllocateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, useDedicated ? &dedicatedInfo : nullptr}; + exportInfo.handleTypes = kMemoryHandle; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo}; + allocate.allocationSize = requirements.memoryRequirements.size; + allocate.memoryTypeIndex = memoryType; + check(vkAllocateMemory(target.device, &allocate, nullptr, &target.memory), "Allocate presentation memory"); + check(vkBindImageMemory(target.device, target.image, target.memory, 0), "Bind presentation memory"); + + createMemory(1, &target.gl_memory); + GLint glDedicated = useDedicated; + memoryParameter(target.gl_memory, GL_DEDICATED_MEMORY_OBJECT_EXT, &glDedicated); +#ifdef _WIN32 + VkMemoryGetWin32HandleInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR}; + memoryInfo.memory = target.memory; + memoryInfo.handleType = kMemoryHandle; + HANDLE memoryHandle = nullptr; + check(getMemoryHandle(target.device, &memoryInfo, &memoryHandle), "Export presentation memory"); + importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryHandle); + // Win32 imports retain the resource, not ownership of the handle. + CloseHandle(memoryHandle); +#else + VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; + memoryInfo.memory = target.memory; + memoryInfo.handleType = kMemoryHandle; + int memoryFd = -1; + check(getMemoryHandle(target.device, &memoryInfo, &memoryFd), "Export presentation memory"); + importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryFd); // GL owns the fd. +#endif + gl->glGenTextures(1, &target.texture); + gl->glBindTexture(GL_TEXTURE_2D, target.texture); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + textureMemory(GL_TEXTURE_2D, 1, GL_RGBA8, width, height, target.gl_memory, 0); + + VkExportSemaphoreCreateInfo exportSemaphore{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; + exportSemaphore.handleTypes = kSemaphoreHandle; + VkSemaphoreCreateInfo semaphoreInfo{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &exportSemaphore}; + createSemaphores(2, target.gl_semaphores); + for (int direction = 0; direction < 2; ++direction) { + check(vkCreateSemaphore(target.device, &semaphoreInfo, nullptr, &target.handoff[direction]), "Create presentation semaphore"); +#ifdef _WIN32 + VkSemaphoreGetWin32HandleInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR}; + handleInfo.semaphore = target.handoff[direction]; + handleInfo.handleType = kSemaphoreHandle; + HANDLE semaphoreHandle = nullptr; + check(getSemaphoreHandle(target.device, &handleInfo, &semaphoreHandle), "Export presentation semaphore"); + importSemaphore(target.gl_semaphores[direction], kGLHandle, semaphoreHandle); + CloseHandle(semaphoreHandle); +#else + VkSemaphoreGetFdInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; + handleInfo.semaphore = target.handoff[direction]; + handleInfo.handleType = kSemaphoreHandle; + int semaphoreFd = -1; + check(getSemaphoreHandle(target.device, &handleInfo, &semaphoreFd), "Export presentation semaphore"); + importSemaphore(target.gl_semaphores[direction], kGLHandle, semaphoreFd); +#endif + } + if (gl->glGetError() != GL_NO_ERROR) throw std::runtime_error("OpenGL external memory import failed"); + + VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + poolInfo.queueFamilyIndex = target.family; + check(vkCreateCommandPool(target.device, &poolInfo, nullptr, &target.pool), "Create presentation command pool"); + VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + allocation.commandPool = target.pool; + allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocation.commandBufferCount = 2; + check(vkAllocateCommandBuffers(target.device, &allocation, target.commands), "Allocate presentation commands"); + VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + check(vkCreateFence(target.device, &fenceInfo, nullptr, &target.fence), "Create presentation fence"); + // The acquire from OpenGL is the same every frame. + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + check(vkBeginCommandBuffer(target.commands[1], &begin), "Begin presentation acquire"); + VkImageMemoryBarrier acquire{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + acquire.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + acquire.oldLayout = acquire.newLayout = VK_IMAGE_LAYOUT_GENERAL; + acquire.srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + acquire.dstQueueFamilyIndex = target.family; + acquire.image = target.image; + acquire.subresourceRange = kColor; + vkCmdPipelineBarrier(target.commands[1], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + 0, 0, nullptr, 0, nullptr, 1, &acquire); + check(vkEndCommandBuffer(target.commands[1]), "End presentation acquire"); + + target.program.reset(new QOpenGLShaderProgram); + if (!target.program->addShaderFromSourceCode(QOpenGLShader::Vertex, + "attribute vec2 vertex; varying vec2 uv; void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}") || + !target.program->addShaderFromSourceCode(QOpenGLShader::Fragment, + "uniform sampler2D video; varying vec2 uv; void main(){gl_FragColor=texture2D(video,uv);}") || !target.program->link()) + throw std::runtime_error("OpenGL presentation shader: " + target.program->log().toStdString()); + const float vertices[] = {-1,-1, 1,-1, -1,1, 1,1}; + GLint previousVertexArray = 0; + gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); + extra->glGenVertexArrays(1, &target.vertex_array); + extra->glBindVertexArray(target.vertex_array); + gl->glGenBuffers(1, &target.buffer); + gl->glBindBuffer(GL_ARRAY_BUFFER, target.buffer); + gl->glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); + const int vertex = target.program->attributeLocation("vertex"); + target.program->enableAttributeArray(vertex); + target.program->setAttributeBuffer(vertex, GL_FLOAT, 0, 2); + gl->glBindBuffer(GL_ARRAY_BUFFER, 0); + extra->glBindVertexArray(previousVertexArray); + } bool draw(int x, int y, int width, int height) { if (qEnvironmentVariable("OPENSHOT_PRESENTATION_BACKEND") == "cpu") return unavailable("OPENSHOT_PRESENTATION_BACKEND=cpu (Vulkan processing remains enabled)"); @@ -102,18 +352,11 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { const auto* value = context->functions()->glGetString(name); return value ? reinterpret_cast(value) : "unknown"; }; -#ifdef _WIN32 - const bool memory = context->hasExtension("GL_EXT_memory_object_win32"); - const bool semaphore = context->hasExtension("GL_EXT_semaphore_win32"); -#else - const bool memory = context->hasExtension("GL_EXT_memory_object_fd"); - const bool semaphore = context->hasExtension("GL_EXT_semaphore_fd"); -#endif Logger::Instance()->Log(std::string("GPU presentation Qt context: vendor=") + text(GL_VENDOR) + "; renderer=" + text(GL_RENDERER) + "; version=" + text(GL_VERSION) + "; OpenGL ES=" + (context->isOpenGLES() ? "yes" : "no") + - "; external memory=" + (memory ? "yes" : "no") + - "; external semaphore=" + (semaphore ? "yes" : "no"), Logger::LevelInfo); + "; external memory=" + (context->hasExtension(kMemoryExtension) ? "yes" : "no") + + "; external semaphore=" + (context->hasExtension(kSemaphoreExtension) ? "yes" : "no"), Logger::LevelInfo); logged_context = context; identity_logged = false; } @@ -144,6 +387,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { // Multi-plane decode surfaces must first go through the RGBA compositor. if (fc->sw_format != AV_PIX_FMT_RGBA || vfc->format[0] != VK_FORMAT_R8G8B8A8_UNORM || image->img[1]) return unavailable("Unsupported Vulkan image format for OpenGL sharing"); + if (!(vfc->usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT)) + return unavailable("Vulkan frame cannot be copied for OpenGL sharing"); VkPhysicalDeviceIDProperties identity{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES}; VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &identity}; vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); @@ -154,11 +399,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { Logger::LevelInfo); identity_logged = true; } -#ifdef _WIN32 - if (!context->hasExtension("GL_EXT_memory_object_win32") || !context->hasExtension("GL_EXT_semaphore_win32")) -#else - if (!context->hasExtension("GL_EXT_memory_object_fd") || !context->hasExtension("GL_EXT_semaphore_fd")) -#endif + if (!context->hasExtension(kMemoryExtension) || !context->hasExtension(kSemaphoreExtension)) return unavailable("OpenGL driver lacks external memory or semaphore sharing extensions"); auto gl = context->functions(); auto extra = context->extraFunctions(); @@ -180,225 +421,110 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { (renderer ? reinterpret_cast(renderer) : "unknown") + "; Vulkan: " + properties.properties.deviceName + ")"); } - auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); - auto deleteMemory = glFunction(context, "glDeleteMemoryObjectsEXT"); - auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); -#ifdef _WIN32 - auto importMemory = glFunction(context, "glImportMemoryWin32HandleEXT"); -#else - auto importMemory = glFunction(context, "glImportMemoryFdEXT"); -#endif - auto textureMemory = glFunction(context, "glTexStorageMem2DEXT"); - auto createSemaphore = glFunction(context, "glGenSemaphoresEXT"); - auto deleteSemaphore = glFunction(context, "glDeleteSemaphoresEXT"); -#ifdef _WIN32 - auto importSemaphore = glFunction(context, "glImportSemaphoreWin32HandleEXT"); -#else - auto importSemaphore = glFunction(context, "glImportSemaphoreFdEXT"); -#endif - auto waitSemaphore = glFunction(context, "glWaitSemaphoreEXT"); - auto signalSemaphore = glFunction(context, "glSignalSemaphoreEXT"); -#ifdef _WIN32 - auto getMemoryHandle = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryWin32HandleKHR")); - auto getSemaphoreHandle = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreWin32HandleKHR")); - if (!getMemoryHandle || !getSemaphoreHandle || vk->queue_family_comp_index < 0) - return unavailable("Vulkan device lacks Win32 memory/semaphore export or a compute queue"); -#else - auto getMemoryFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetMemoryFdKHR")); - auto getSemaphoreFd = reinterpret_cast(vkGetDeviceProcAddr(vk->act_dev, "vkGetSemaphoreFdKHR")); - if (!getMemoryFd || !getSemaphoreFd || vk->queue_family_comp_index < 0) - return unavailable("Vulkan device lacks file-descriptor memory/semaphore export or a compute queue"); -#endif - // Construct all fallible GL resources before transferring image ownership. - QOpenGLShaderProgram program; - if (!program.addShaderFromSourceCode(QOpenGLShader::Vertex, - "attribute vec2 vertex; varying vec2 uv; void main(){uv=vec2((vertex.x+1.0)*0.5,(1.0-vertex.y)*0.5);gl_Position=vec4(vertex,0,1);}") || - !program.addShaderFromSourceCode(QOpenGLShader::Fragment, - "uniform sampler2D video; varying vec2 uv; void main(){gl_FragColor=texture2D(video,uv);}") || !program.link()) return unavailable("OpenGL presentation shader: " + program.log().toStdString()); - stage("shader"); - VkDevice device = vk->act_dev; - GLuint memory = 0, texture = 0, semaphores[2] = {0, 0}, buffer = 0, vertexArray = 0; - GLint previousVertexArray = 0; - gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); - VkSemaphore handoff[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; - VkCommandPool pool = VK_NULL_HANDLE; - VkFence fence = VK_NULL_HANDLE; -#ifdef _WIN32 - HANDLE memoryHandle = nullptr, semaphoreHandle = nullptr; -#else - int memoryFd = -1, semaphoreFd = -1; -#endif - auto cleanup = [&]() { - if (vertexArray) { - extra->glBindVertexArray(previousVertexArray); - extra->glDeleteVertexArrays(1, &vertexArray); - } - if (buffer) gl->glDeleteBuffers(1, &buffer); - if (texture) gl->glDeleteTextures(1, &texture); - stage("delete texture"); - if (memory) deleteMemory(1, &memory); - stage("delete memory"); - deleteSemaphore(2, semaphores); - stage("delete GL semaphore"); - if (fence) vkDestroyFence(device, fence, nullptr); - if (pool) vkDestroyCommandPool(device, pool, nullptr); - for (auto semaphore : handoff) - if (semaphore) vkDestroySemaphore(device, semaphore, nullptr); - stage("delete Vulkan resources"); -#ifdef _WIN32 - if (memoryHandle) CloseHandle(memoryHandle); - if (semaphoreHandle) CloseHandle(semaphoreHandle); -#else - if (memoryFd >= 0) close(memoryFd); - if (semaphoreFd >= 0) close(semaphoreFd); -#endif - }; - bool locked = false; + if (vk->queue_family_comp_index < 0) + return unavailable("Vulkan device lacks a compute queue"); + + bool locked = false, copySubmitted = false; + uint64_t copyValue = 0; try { -#ifdef _WIN32 - VkMemoryGetWin32HandleInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR}; - memoryInfo.memory = image->mem[0]; - memoryInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT; - check(getMemoryHandle(device, &memoryInfo, &memoryHandle), "Export Vulkan image memory"); -#else - VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; - memoryInfo.memory = image->mem[0]; - memoryInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; - check(getMemoryFd(device, &memoryInfo, &memoryFd), "Export Vulkan image memory"); -#endif - createMemory(1, &memory); - VkMemoryDedicatedRequirements dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; - VkMemoryRequirements2 requirements{VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated}; - VkImageMemoryRequirementsInfo2 requirementInfo{VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2}; - requirementInfo.image = image->img[0]; - vkGetImageMemoryRequirements2(device, &requirementInfo, &requirements); - GLint useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; - memoryParameter(memory, GL_DEDICATED_MEMORY_OBJECT_EXT, &useDedicated); -#ifdef _WIN32 - importMemory(memory, image->size[0], GL_HANDLE_TYPE_OPAQUE_WIN32_EXT, memoryHandle); - // Unlike file descriptors, Win32 imports retain the resource, not - // ownership of the application's handle. - CloseHandle(memoryHandle); - memoryHandle = nullptr; -#else - importMemory(memory, image->size[0], GL_HANDLE_TYPE_OPAQUE_FD_EXT, memoryFd); - memoryFd = -1; // GL consumes imported descriptors. -#endif - stage("import memory"); - gl->glGenTextures(1, &texture); - gl->glBindTexture(GL_TEXTURE_2D, texture); - gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - textureMemory(GL_TEXTURE_2D, 1, GL_RGBA8, source->width, source->height, memory, image->offset[0]); - stage("texture storage"); - VkExportSemaphoreCreateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; -#ifdef _WIN32 - exportInfo.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; -#else - exportInfo.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; -#endif - VkSemaphoreCreateInfo semaphoreInfo{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &exportInfo}; - // Separate directions: reusing one binary semaphore lets Vulkan's - // acquire consume its own release signal before GL can wait on it. - createSemaphore(2, semaphores); - for (int direction = 0; direction < 2; ++direction) { - check(vkCreateSemaphore(device, &semaphoreInfo, nullptr, &handoff[direction]), "Create presentation semaphore"); -#ifdef _WIN32 - VkSemaphoreGetWin32HandleInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR}; - handleInfo.semaphore = handoff[direction]; - handleInfo.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; - check(getSemaphoreHandle(device, &handleInfo, &semaphoreHandle), "Export presentation semaphore"); - importSemaphore(semaphores[direction], GL_HANDLE_TYPE_OPAQUE_WIN32_EXT, semaphoreHandle); - CloseHandle(semaphoreHandle); - semaphoreHandle = nullptr; -#else - VkSemaphoreGetFdInfoKHR fdInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; - fdInfo.semaphore = handoff[direction]; - fdInfo.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - check(getSemaphoreFd(device, &fdInfo, &semaphoreFd), "Export presentation semaphore"); - importSemaphore(semaphores[direction], GL_HANDLE_TYPE_OPAQUE_FD_EXT, semaphoreFd); - semaphoreFd = -1; -#endif - } - if (gl->glGetError() != GL_NO_ERROR) throw std::runtime_error("OpenGL external memory import failed"); - const uint32_t family = vk->queue_family_comp_index; - VkQueue queue; - vkGetDeviceQueue(device, family, 0, &queue); - VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; - poolInfo.queueFamilyIndex = family; - check(vkCreateCommandPool(device, &poolInfo, nullptr, &pool), "Create presentation command pool"); - VkCommandBuffer commands[2]; - VkCommandBufferAllocateInfo allocation{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - allocation.commandPool = pool; - allocation.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - allocation.commandBufferCount = 2; - check(vkAllocateCommandBuffers(device, &allocation, commands), "Allocate presentation commands"); - VkFenceCreateInfo fenceInfo{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - check(vkCreateFence(device, &fenceInfo, nullptr, &fence), "Create presentation fence"); - const float vertices[] = {-1,-1, 1,-1, -1,1, 1,1}; - extra->glGenVertexArrays(1, &vertexArray); - extra->glBindVertexArray(vertexArray); - gl->glGenBuffers(1, &buffer); - gl->glBindBuffer(GL_ARRAY_BUFFER, buffer); - gl->glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW); - program.bind(); - program.setUniformValue("video", 0); - const int vertex = program.attributeLocation("vertex"); - program.enableAttributeArray(vertex); - program.setAttributeBuffer(vertex, GL_FLOAT, 0, 2); - vfc->lock_frame(fc, image); - locked = true; - stage("frame lock"); - const auto oldLayout = image->layout[0]; - const auto oldOwner = image->queue_family[0]; - for (int i = 0; i < 2; ++i) { - VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - check(vkBeginCommandBuffer(commands[i], &begin), "Begin presentation commands"); - VkImageMemoryBarrier barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; - barrier.srcAccessMask = i ? 0 : image->access[0]; - barrier.dstAccessMask = i ? VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT : 0; - barrier.oldLayout = i ? VK_IMAGE_LAYOUT_GENERAL : oldLayout; - barrier.newLayout = VK_IMAGE_LAYOUT_GENERAL; - barrier.srcQueueFamilyIndex = i ? VK_QUEUE_FAMILY_EXTERNAL : oldOwner; - barrier.dstQueueFamilyIndex = i ? family : VK_QUEUE_FAMILY_EXTERNAL; - barrier.image = image->img[0]; - barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - vkCmdPipelineBarrier(commands[i], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); - check(vkEndCommandBuffer(commands[i]), "End presentation commands"); + if (target.context != context || target.device != vk->act_dev || + target.width != source->width || target.height != source->height) { + releaseTarget(); + createTarget(context, fc, source->width, source->height); + stage("create shared image"); } - auto submit = [&](int index, VkSemaphore wait, uint64_t waitValue, VkSemaphore signal, uint64_t signalValue) { - VkTimelineSemaphoreSubmitInfo timeline{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; - timeline.waitSemaphoreValueCount = timeline.signalSemaphoreValueCount = 1; - timeline.pWaitSemaphoreValues = &waitValue; - timeline.pSignalSemaphoreValues = &signalValue; - VkPipelineStageFlags stage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; - VkSubmitInfo info{VK_STRUCTURE_TYPE_SUBMIT_INFO, &timeline}; - info.waitSemaphoreCount = info.signalSemaphoreCount = info.commandBufferCount = 1; - info.pWaitSemaphores = &wait; - info.pWaitDstStageMask = &stage; - info.pSignalSemaphores = &signal; - info.pCommandBuffers = &commands[index]; - vk->lock_queue(dc, family, 0); - auto result = vkQueueSubmit(queue, 1, &info, index ? fence : VK_NULL_HANDLE); - vk->unlock_queue(dc, family, 0); + // The previous frame's commands must retire before re-recording. + waitPending(); + stage("previous frame"); + const auto submit = [&](const VkSubmitInfo& info, VkFence signalFence) { + vk->lock_queue(dc, target.family, 0); + const auto result = vkQueueSubmit(target.queue, 1, &info, signalFence); + vk->unlock_queue(dc, target.family, 0); check(result, "Submit presentation transfer"); }; - submit(0, image->sem[0], image->sem_value[0], handoff[0], 0); - stage("release submit"); + vfc->lock_frame(fc, image); + locked = true; + VkCommandBuffer copy = target.commands[0]; + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + check(vkBeginCommandBuffer(copy, &begin), "Begin presentation copy"); + // FFmpeg frames are concurrent or owned by the compute family, so + // the copy needs no ownership transfer of the source. + VkImageMemoryBarrier toCopy[2] = {{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}, {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}}; + toCopy[0].srcAccessMask = image->access[0]; + toCopy[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toCopy[0].oldLayout = image->layout[0]; + toCopy[0].newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toCopy[0].srcQueueFamilyIndex = toCopy[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toCopy[0].image = image->img[0]; + toCopy[0].subresourceRange = kColor; + toCopy[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toCopy[1].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toCopy[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toCopy[1].srcQueueFamilyIndex = toCopy[1].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toCopy[1].image = target.image; + toCopy[1].subresourceRange = kColor; + vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 2, toCopy); + VkImageCopy region{}; + region.srcSubresource = region.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.extent = {static_cast(target.width), static_cast(target.height), 1}; + vkCmdCopyImage(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + target.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + VkImageMemoryBarrier release{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + release.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + release.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + release.newLayout = VK_IMAGE_LAYOUT_GENERAL; + release.srcQueueFamilyIndex = target.family; + release.dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + release.image = target.image; + release.subresourceRange = kColor; + vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + 0, 0, nullptr, 0, nullptr, 1, &release); + check(vkEndCommandBuffer(copy), "End presentation copy"); + // Advance the frame's timeline and release the copy to OpenGL. + const uint64_t waitValue = image->sem_value[0]; + const uint64_t signalValues[2] = {image->sem_value[0] + 1, 0}; + const VkSemaphore copySignals[2] = {image->sem[0], target.handoff[0]}; + VkTimelineSemaphoreSubmitInfo timeline{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; + timeline.waitSemaphoreValueCount = 1; + timeline.pWaitSemaphoreValues = &waitValue; + timeline.signalSemaphoreValueCount = 2; + timeline.pSignalSemaphoreValues = signalValues; + const VkPipelineStageFlags waitStage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkSubmitInfo copyInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO, &timeline}; + copyInfo.waitSemaphoreCount = 1; + copyInfo.pWaitSemaphores = &image->sem[0]; + copyInfo.pWaitDstStageMask = &waitStage; + copyInfo.commandBufferCount = 1; + copyInfo.pCommandBuffers = © + copyInfo.signalSemaphoreCount = 2; + copyInfo.pSignalSemaphores = copySignals; + submit(copyInfo, VK_NULL_HANDLE); + copySubmitted = true; + copyValue = ++image->sem_value[0]; + image->layout[0] = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + image->access[0] = VK_ACCESS_TRANSFER_READ_BIT; + vfc->unlock_frame(fc, image); + locked = false; + stage("copy submit"); + const GLenum layout = GL_LAYOUT_GENERAL_EXT; - waitSemaphore(semaphores[0], 0, nullptr, 1, &texture, &layout); - GLint viewport[4]; + target.wait_semaphore(target.gl_semaphores[0], 0, nullptr, 1, &target.texture, &layout); + GLint viewport[4], previousVertexArray = 0; gl->glGetIntegerv(GL_VIEWPORT, viewport); + gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); gl->glViewport(x, y, width, height); gl->glActiveTexture(GL_TEXTURE0); - gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glBindTexture(GL_TEXTURE_2D, target.texture); gl->glDisable(GL_DEPTH_TEST); gl->glDisable(GL_SCISSOR_TEST); gl->glDisable(GL_STENCIL_TEST); gl->glDisable(GL_BLEND); gl->glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + target.program->bind(); + target.program->setUniformValue("video", 0); + extra->glBindVertexArray(target.vertex_array); gl->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); if (trace) { const auto error = gl->glGetError(); @@ -408,35 +534,36 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &boundTexture); qWarning() << "GPU presentation draw state: error=" << error << "framebuffer=" << framebuffer << "program=" << activeProgram - << "texture=" << boundTexture << "vertex=" << vertex; + << "texture=" << boundTexture; } - signalSemaphore(semaphores[1], 0, nullptr, 1, &texture, &layout); + extra->glBindVertexArray(previousVertexArray); + target.program->release(); + gl->glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); + target.signal_semaphore(target.gl_semaphores[1], 0, nullptr, 1, &target.texture, &layout); gl->glFlush(); stage("GL draw flush"); - submit(1, handoff[1], 0, image->sem[0], image->sem_value[0] + 1); - image->sem_value[0]++; - image->layout[0] = VK_IMAGE_LAYOUT_GENERAL; - image->access[0] = VK_ACCESS_MEMORY_READ_BIT; - image->queue_family[0] = family; - vfc->unlock_frame(fc, image); - locked = false; - // Synchronous prototype: no resource reuse before GL and Vulkan finish. - check(vkWaitForFences(device, 1, &fence, VK_TRUE, UINT64_MAX), "Wait for presentation transfer"); - stage("acquire fence"); - gl->glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); - program.disableAttributeArray(vertex); - program.release(); - gl->glBindBuffer(GL_ARRAY_BUFFER, 0); - // Retire GL's imported-object command references as well as the - // Vulkan queue work before destroying the cross-API resources. - gl->glFinish(); - stage("GL finish"); - cleanup(); - stage("cleanup"); + + VkSubmitInfo acquireInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + acquireInfo.waitSemaphoreCount = 1; + acquireInfo.pWaitSemaphores = &target.handoff[1]; + acquireInfo.pWaitDstStageMask = &waitStage; + acquireInfo.commandBufferCount = 1; + acquireInfo.pCommandBuffers = &target.commands[1]; + submit(acquireInfo, target.fence); + target.pending = true; + stage("acquire submit"); return true; } catch (...) { if (locked) vfc->unlock_frame(fc, image); - cleanup(); + // A half-finished handoff leaves the semaphores unusable. + if (copySubmitted && !target.pending) { + VkSemaphoreWaitInfo copyDone{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + copyDone.semaphoreCount = 1; + copyDone.pSemaphores = &image->sem[0]; + copyDone.pValues = ©Value; + vkWaitSemaphores(target.device, ©Done, UINT64_MAX); + } + releaseTarget(); throw; } } diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index c34c03d9b..1c0652022 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -175,6 +175,23 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ CHECK(openshot::VulkanFrameReadbackCount() == readbacks); CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); + // A new frame size replaces the presenter's shared OpenGL image. + QImage resized(96, 48, QImage::Format_RGBA8888_Premultiplied); + for (int y = 0; y < resized.height(); ++y) + for (int x = 0; x < resized.width(); ++x) + resized.setPixelColor(x, y, y < resized.height() / 2 ? Qt::blue : Qt::red); + frame->AddDeferredImage(openshot::RenderVulkanImage(openshot::UploadVulkanImage(resized), nullptr, + 96, 48, QTransform(), 1.0, Qt::transparent)); + renderer.paint(frame); + for (int i = 0; i < 20; ++i) { + QCoreApplication::processEvents(); + QThread::msleep(5); + } + image = widget.grabFramebuffer(); + CHECK(widget.nativeDraw); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::blue)); + CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::red)); widget.presenter.Clear(); CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); CHECK_FALSE(widget.presenter.FallbackReason().empty()); From bffc1f617c6cfe4528f2413b7aefa7329c6d1b02 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 15:40:33 -0600 Subject: [PATCH 27/50] Share presentation frames through a linear buffer and skip GPU teardown at exit Maxwell GPUs raise a TEX LAYOUT exception, then a driver reset, when OpenGL samples an imported optimal-tiling Vulkan image. Copy each frame into an exported buffer instead and let OpenGL unpack it into its own texture. The shared Vulkan device and compositor output pools are no longer freed by static destructors, which ran during DLL unload after the NVIDIA driver had begun shutting down. --- src/GpuDevices.cpp | 5 +- src/Qt/VulkanOpenGLPresenter.cpp | 132 ++++++++++++++++--------------- src/VulkanCompositor.cpp | 5 +- 3 files changed, 75 insertions(+), 67 deletions(-) diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 8ad4a2328..1c7080d6f 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -200,13 +200,14 @@ bool EncoderOpens(const char* name, AVBufferRef* device, AVPixelFormat hw_format int CreateSharedVulkanDevice(AVBufferRef** output) { // All readers share one device so their surfaces can be composed without interop. // Keep an owning reference for the process lifetime; each caller gets its own ref. + // Never destroyed: destroying the device during process exit crashes drivers + // that are already unloading. struct SharedDevice { std::mutex mutex; AVBufferRef* device = nullptr; bool tried = false; - ~SharedDevice() { av_buffer_unref(&device); } }; - static SharedDevice shared; + static auto& shared = *new SharedDevice; const std::lock_guard lock(shared.mutex); if (!shared.device) { // Only search once; without a usable GPU every reader decodes on the CPU. diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index b47be21df..c5ca42a94 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -61,11 +61,13 @@ using ImportSemaphore = PFNGLIMPORTSEMAPHOREFDEXTPROC; #endif constexpr VkImageSubresourceRange kColor = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; } -// OpenGL samples a presenter-owned Vulkan image rather than the frame itself. -// NVIDIA's Windows driver returns zeros when FFmpeg or compositor frames are -// read through an exported alias of their memory (even from Vulkan), while a -// GPU copy into an image allocated for export is visible. The shared image and -// its OpenGL import persist until the frame size, device, or context changes. +// Frames reach OpenGL through a presenter-owned linear buffer, never through +// shared image memory. NVIDIA's Windows driver returns zeros when FFmpeg or +// compositor frames are read through an exported alias of their memory, and +// Maxwell GPUs raise a TEX LAYOUT exception (then a TDR) when OpenGL samples +// an imported optimal-tiling Vulkan image. Vulkan copies each frame into the +// buffer, and OpenGL unpacks it into its own texture. The buffer and its +// OpenGL import persist until the frame size, device, or context changes. struct SharedTarget { AVBufferRef* device_ref = nullptr; VkDevice device = VK_NULL_HANDLE; @@ -73,7 +75,7 @@ struct SharedTarget { int width = 0, height = 0; uint32_t family = 0; VkQueue queue = VK_NULL_HANDLE; - VkImage image = VK_NULL_HANDLE; + VkBuffer pixels = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; // Separate directions: reusing one binary semaphore lets Vulkan's // acquire consume its own release signal before GL can wait on it. @@ -82,7 +84,7 @@ struct SharedTarget { VkCommandBuffer commands[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; VkFence fence = VK_NULL_HANDLE; bool pending = false; - GLuint gl_memory = 0, texture = 0, gl_semaphores[2] = {0, 0}, buffer = 0, vertex_array = 0; + GLuint gl_memory = 0, gl_pixels = 0, texture = 0, gl_semaphores[2] = {0, 0}, buffer = 0, vertex_array = 0; std::unique_ptr program; PFNGLWAITSEMAPHOREEXTPROC wait_semaphore = nullptr; PFNGLSIGNALSEMAPHOREEXTPROC signal_semaphore = nullptr; @@ -147,6 +149,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { auto gl = context->functions(); if (target.vertex_array) context->extraFunctions()->glDeleteVertexArrays(1, &target.vertex_array); if (target.buffer) gl->glDeleteBuffers(1, &target.buffer); + if (target.gl_pixels) gl->glDeleteBuffers(1, &target.gl_pixels); if (target.texture) gl->glDeleteTextures(1, &target.texture); if (target.gl_memory) reinterpret_cast(context->getProcAddress("glDeleteMemoryObjectsEXT"))(1, &target.gl_memory); @@ -159,7 +162,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (target.pool) vkDestroyCommandPool(target.device, target.pool, nullptr); for (auto semaphore : target.handoff) if (semaphore) vkDestroySemaphore(target.device, semaphore, nullptr); - if (target.image) vkDestroyImage(target.device, target.image, nullptr); + if (target.pixels) vkDestroyBuffer(target.device, target.pixels, nullptr); if (target.memory) vkFreeMemory(target.device, target.memory, nullptr); } av_buffer_unref(&target.device_ref); @@ -179,7 +182,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { auto extra = context->extraFunctions(); auto createMemory = glFunction(context, "glCreateMemoryObjectsEXT"); auto memoryParameter = glFunction(context, "glMemoryObjectParameterivEXT"); - auto textureMemory = glFunction(context, "glTexStorageMem2DEXT"); + auto bufferMemory = glFunction(context, "glBufferStorageMemEXT"); auto createSemaphores = glFunction(context, "glGenSemaphoresEXT"); glFunction(context, "glDeleteMemoryObjectsEXT"); glFunction(context, "glDeleteSemaphoresEXT"); @@ -199,24 +202,19 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (!getMemoryHandle || !getSemaphoreHandle) throw std::runtime_error("Vulkan device lacks external memory/semaphore export"); - VkExternalMemoryImageCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO}; + const VkDeviceSize size = VkDeviceSize(width) * height * 4; + VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO}; external.handleTypes = kMemoryHandle; - VkImageCreateInfo create{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, &external}; - create.imageType = VK_IMAGE_TYPE_2D; - create.format = VK_FORMAT_R8G8B8A8_UNORM; - create.extent = {static_cast(width), static_cast(height), 1}; - create.mipLevels = create.arrayLayers = 1; - create.samples = VK_SAMPLE_COUNT_1_BIT; - create.tiling = VK_IMAGE_TILING_OPTIMAL; - create.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; + VkBufferCreateInfo create{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; + create.size = size; + create.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; create.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - create.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - check(vkCreateImage(target.device, &create, nullptr, &target.image), "Create presentation image"); + check(vkCreateBuffer(target.device, &create, nullptr, &target.pixels), "Create presentation buffer"); VkMemoryDedicatedRequirements dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS}; VkMemoryRequirements2 requirements{VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, &dedicated}; - VkImageMemoryRequirementsInfo2 requirementInfo{VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2}; - requirementInfo.image = target.image; - vkGetImageMemoryRequirements2(target.device, &requirementInfo, &requirements); + VkBufferMemoryRequirementsInfo2 requirementInfo{VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2}; + requirementInfo.buffer = target.pixels; + vkGetBufferMemoryRequirements2(target.device, &requirementInfo, &requirements); VkPhysicalDeviceMemoryProperties memoryProperties; vkGetPhysicalDeviceMemoryProperties(vk->phys_dev, &memoryProperties); uint32_t memoryType = UINT32_MAX; @@ -227,14 +225,14 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (memoryType == UINT32_MAX) throw std::runtime_error("No device-local presentation memory"); const bool useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; VkMemoryDedicatedAllocateInfo dedicatedInfo{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO}; - dedicatedInfo.image = target.image; + dedicatedInfo.buffer = target.pixels; VkExportMemoryAllocateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, useDedicated ? &dedicatedInfo : nullptr}; exportInfo.handleTypes = kMemoryHandle; VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &exportInfo}; allocate.allocationSize = requirements.memoryRequirements.size; allocate.memoryTypeIndex = memoryType; check(vkAllocateMemory(target.device, &allocate, nullptr, &target.memory), "Allocate presentation memory"); - check(vkBindImageMemory(target.device, target.image, target.memory, 0), "Bind presentation memory"); + check(vkBindBufferMemory(target.device, target.pixels, target.memory, 0), "Bind presentation memory"); createMemory(1, &target.gl_memory); GLint glDedicated = useDedicated; @@ -256,13 +254,21 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { check(getMemoryHandle(target.device, &memoryInfo, &memoryFd), "Export presentation memory"); importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryFd); // GL owns the fd. #endif + GLint previousUnpack = 0, previousTexture = 0; + gl->glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previousUnpack); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &previousTexture); + gl->glGenBuffers(1, &target.gl_pixels); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, target.gl_pixels); + bufferMemory(GL_PIXEL_UNPACK_BUFFER, size, target.gl_memory, 0); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, previousUnpack); gl->glGenTextures(1, &target.texture); gl->glBindTexture(GL_TEXTURE_2D, target.texture); gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - textureMemory(GL_TEXTURE_2D, 1, GL_RGBA8, width, height, target.gl_memory, 0); + gl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + gl->glBindTexture(GL_TEXTURE_2D, previousTexture); VkExportSemaphoreCreateInfo exportSemaphore{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; exportSemaphore.handleTypes = kSemaphoreHandle; @@ -303,15 +309,14 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { // The acquire from OpenGL is the same every frame. VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; check(vkBeginCommandBuffer(target.commands[1], &begin), "Begin presentation acquire"); - VkImageMemoryBarrier acquire{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + VkBufferMemoryBarrier acquire{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; acquire.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - acquire.oldLayout = acquire.newLayout = VK_IMAGE_LAYOUT_GENERAL; acquire.srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; acquire.dstQueueFamilyIndex = target.family; - acquire.image = target.image; - acquire.subresourceRange = kColor; + acquire.buffer = target.pixels; + acquire.size = VK_WHOLE_SIZE; vkCmdPipelineBarrier(target.commands[1], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - 0, 0, nullptr, 0, nullptr, 1, &acquire); + 0, 0, nullptr, 1, &acquire, 0, nullptr); check(vkEndCommandBuffer(target.commands[1]), "End presentation acquire"); target.program.reset(new QOpenGLShaderProgram); @@ -431,7 +436,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { target.width != source->width || target.height != source->height) { releaseTarget(); createTarget(context, fc, source->width, source->height); - stage("create shared image"); + stage("create shared buffer"); } // The previous frame's commands must retire before re-recording. waitPending(); @@ -450,37 +455,28 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { check(vkBeginCommandBuffer(copy, &begin), "Begin presentation copy"); // FFmpeg frames are concurrent or owned by the compute family, so // the copy needs no ownership transfer of the source. - VkImageMemoryBarrier toCopy[2] = {{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}, {VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}}; - toCopy[0].srcAccessMask = image->access[0]; - toCopy[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - toCopy[0].oldLayout = image->layout[0]; - toCopy[0].newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; - toCopy[0].srcQueueFamilyIndex = toCopy[0].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - toCopy[0].image = image->img[0]; - toCopy[0].subresourceRange = kColor; - toCopy[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - toCopy[1].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - toCopy[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - toCopy[1].srcQueueFamilyIndex = toCopy[1].dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; - toCopy[1].image = target.image; - toCopy[1].subresourceRange = kColor; + VkImageMemoryBarrier toCopy{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toCopy.srcAccessMask = image->access[0]; + toCopy.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toCopy.oldLayout = image->layout[0]; + toCopy.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + toCopy.srcQueueFamilyIndex = toCopy.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toCopy.image = image->img[0]; + toCopy.subresourceRange = kColor; vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, 0, nullptr, 0, nullptr, 2, toCopy); - VkImageCopy region{}; - region.srcSubresource = region.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; - region.extent = {static_cast(target.width), static_cast(target.height), 1}; - vkCmdCopyImage(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - target.image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); - VkImageMemoryBarrier release{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + 0, 0, nullptr, 0, nullptr, 1, &toCopy); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {static_cast(target.width), static_cast(target.height), 1}; + vkCmdCopyImageToBuffer(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.pixels, 1, ®ion); + VkBufferMemoryBarrier release{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; release.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - release.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - release.newLayout = VK_IMAGE_LAYOUT_GENERAL; release.srcQueueFamilyIndex = target.family; release.dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; - release.image = target.image; - release.subresourceRange = kColor; + release.buffer = target.pixels; + release.size = VK_WHOLE_SIZE; vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - 0, 0, nullptr, 0, nullptr, 1, &release); + 0, 0, nullptr, 1, &release, 0, nullptr); check(vkEndCommandBuffer(copy), "End presentation copy"); // Advance the frame's timeline and release the copy to OpenGL. const uint64_t waitValue = image->sem_value[0]; @@ -509,14 +505,23 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { locked = false; stage("copy submit"); - const GLenum layout = GL_LAYOUT_GENERAL_EXT; - target.wait_semaphore(target.gl_semaphores[0], 0, nullptr, 1, &target.texture, &layout); - GLint viewport[4], previousVertexArray = 0; + target.wait_semaphore(target.gl_semaphores[0], 1, &target.gl_pixels, 0, nullptr, nullptr); + GLint viewport[4], previousVertexArray = 0, previousUnpack = 0, unpackAlignment = 4, unpackRowLength = 0; gl->glGetIntegerv(GL_VIEWPORT, viewport); gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); - gl->glViewport(x, y, width, height); + gl->glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previousUnpack); + gl->glGetIntegerv(GL_UNPACK_ALIGNMENT, &unpackAlignment); + gl->glGetIntegerv(GL_UNPACK_ROW_LENGTH, &unpackRowLength); gl->glActiveTexture(GL_TEXTURE0); gl->glBindTexture(GL_TEXTURE_2D, target.texture); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, target.gl_pixels); + gl->glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + gl->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, target.width, target.height, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, previousUnpack); + gl->glPixelStorei(GL_UNPACK_ALIGNMENT, unpackAlignment); + gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, unpackRowLength); + gl->glViewport(x, y, width, height); gl->glDisable(GL_DEPTH_TEST); gl->glDisable(GL_SCISSOR_TEST); gl->glDisable(GL_STENCIL_TEST); @@ -539,7 +544,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { extra->glBindVertexArray(previousVertexArray); target.program->release(); gl->glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); - target.signal_semaphore(target.gl_semaphores[1], 0, nullptr, 1, &target.texture, &layout); + // Signals once the unpack above has finished reading the buffer. + target.signal_semaphore(target.gl_semaphores[1], 1, &target.gl_pixels, 0, nullptr, nullptr); gl->glFlush(); stage("GL draw flush"); diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index b51005b9b..4c8fa6016 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -44,9 +44,10 @@ constexpr int kSlots=4; struct OutputPools { std::mutex mutex; std::map,AVBufferRef*> pools; - ~OutputPools(){ for(auto& item:pools) av_buffer_unref(&item.second); } }; -OutputPools& SharedPools(){ static OutputPools pools; return pools; } +// Never destroyed: at process exit the Windows NVIDIA driver may already be +// unloading, and freeing Vulkan frames from a static destructor crashes it. +OutputPools& SharedPools(){ static auto* pools=new OutputPools; return *pools; } constexpr VkImageUsageFlags kOutputUsage=VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USAGE_STORAGE_BIT|VK_IMAGE_USAGE_TRANSFER_SRC_BIT|VK_IMAGE_USAGE_TRANSFER_DST_BIT; // Returns a new reference the caller must release. AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ From f5d8294e0b36628e834a71862c80597f65934958 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 16:50:16 -0600 Subject: [PATCH 28/50] Keep the Vulkan upload cache alive through process exit Its static destructor freed cached Vulkan frames during DLL unload, after the NVIDIA driver had begun shutting down, crashing OpenShot on exit. --- src/VulkanFrameImage.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 1ff8333ee..628d122b6 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -155,7 +155,9 @@ struct UploadCache { int64_t bytes = 0; uint64_t clock = 0; }; -UploadCache& Uploads() { static UploadCache cache; return cache; } +// Never destroyed: freeing cached Vulkan frames from a static destructor at +// process exit crashes the Windows NVIDIA driver, which may already be unloading. +UploadCache& Uploads() { static auto* cache = new UploadCache; return *cache; } struct BufferDeleter { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; using OwnedBuffer = std::unique_ptr; } From 67c636ff738cd12203b9700d3ff65f9d17bb00c0 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 23:08:43 -0600 Subject: [PATCH 29/50] Fix GPU upload leaks and race, and report GPU failures clearly Reuse one Vulkan frame pool per image size; a pool per upload leaked device memory. Share still and cover-art images between frames instead of deep copying them, so they are uploaded once instead of every frame. Wait for FFmpeg's upload to finish before freeing the CPU pixels it may read directly through host-mapped memory. Name the VkResult in presenter and compositor errors, count failed GPU uploads (logged every 100th) and stop reporting CPU-decoded cover art as missing GPU decode. --- bindings/python/openshot.i | 2 +- src/Clip.cpp | 10 +++-- src/FFmpegReader.cpp | 29 +++++++------- src/GpuDevices.cpp | 25 ++++++++++++ src/GpuDevices.h | 2 + src/GpuMetrics.cpp | 2 +- src/GpuMetrics.h | 2 +- src/Qt/VulkanOpenGLPresenter.cpp | 3 +- src/VulkanCompositor.cpp | 3 +- src/VulkanFrameImage.cpp | 65 +++++++++++++++++++++++++------- src/VulkanFrameImage.h | 5 +++ tests/VulkanCompositor.cpp | 27 +++++++++++++ 12 files changed, 140 insertions(+), 35 deletions(-) diff --git a/bindings/python/openshot.i b/bindings/python/openshot.i index c7b3cfddb..99c6b3680 100644 --- a/bindings/python/openshot.i +++ b/bindings/python/openshot.i @@ -529,7 +529,7 @@ static int openshot_swig_is_qwidget(PyObject *obj) { %include "RendererBase.h" #if defined(OPENSHOT_ENABLE_GPU_FRAME) %include "Qt/VulkanOpenGLPresenter.h" -namespace openshot { void ReleaseVulkanMemory(); uint64_t VulkanFrameReadbackCount(); } +namespace openshot { void ReleaseVulkanMemory(); uint64_t VulkanFrameReadbackCount(); uint64_t VulkanUploadFailureCount(); } %include "GpuDevices.h" %template(GpuDeviceVector) std::vector; %template(StringVector) std::vector; diff --git a/src/Clip.cpp b/src/Clip.cpp index 7f2b95bf7..712b050b5 100644 --- a/src/Clip.cpp +++ b/src/Clip.cpp @@ -1462,9 +1462,13 @@ std::shared_ptr VulkanSource(const std::shared_ptrAddDeferredImage(uploaded); return std::dynamic_pointer_cast(uploaded); } catch (const std::exception& error) { - static std::atomic_flag warned = ATOMIC_FLAG_INIT; - if (!warned.test_and_set()) - Logger::Instance()->Log(std::string("GPU image upload unavailable; using CPU: ") + error.what(), Logger::LevelWarning); + // Log the first failure and then every 100th, so repeats stay visible + // without flooding the log; the total is in VulkanUploadFailureCount(). + RecordVulkanUploadFailure(); + const auto failures = VulkanUploadFailureCount(); + if (failures == 1 || failures % 100 == 0) + Logger::Instance()->Log("GPU image upload unavailable; using CPU (" + std::to_string(failures) + + " failures): " + error.what(), Logger::LevelWarning); return {}; } } diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 5f8910a30..66e57bd9f 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -1540,7 +1540,7 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { if (auto surface = frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(frame->GetImage()->copy())); + f->AddImage(std::make_shared(*frame->GetImage())); } // Use solid color (if no image data found) @@ -1562,14 +1562,14 @@ std::shared_ptr FFmpegReader::ReadStream(int64_t requested_frame) { if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } else if (last_video_frame && last_video_frame->has_image_data && last_video_frame->number <= requested_frame) { #ifdef OPENSHOT_ENABLE_GPU_FRAME if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } else { f->AddColor(info.width, info.height, "#000"); } @@ -1819,7 +1819,8 @@ bool FFmpegReader::GetAVFrame() { ? gpu_metrics::DecodeGpu : gpu_metrics::DecodeCpu); packet_status.video_decoded++; #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (packet_status.video_decoded == 1 && !hw_decode_succeeded) + // Cover art and still images are decoded on the CPU by design. + if (packet_status.video_decoded == 1 && !hw_decode_succeeded && !info.has_single_image) Logger::Instance()->Log("GPU decode not observed: CPU video frame received; codec=" + info.vcodec + "; " + path, Logger::LevelInfo); #endif @@ -3001,7 +3002,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); + f->AddImage(std::make_shared(*previous_frame_instance->GetImage())); } // Fall back to last finalized timeline image (survives cache churn). @@ -3013,7 +3014,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } // Fall back to the last decoded image only when it is not from the future. @@ -3025,7 +3026,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } // Last-resort fallback if no prior image is available. @@ -3076,13 +3077,15 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (info.has_single_image) { // For still-image video (including attached cover art), reuse the most // recent image so playback does not stall waiting for video EOF. + // QImage copies share pixels until written, and keep the cache key + // that lets the GPU reuse one upload instead of one per frame. std::shared_ptr previous_frame_instance = final_cache.GetFrame(f->number - 1); if (previous_frame_instance && previous_frame_instance->has_image_data) { #ifdef OPENSHOT_ENABLE_GPU_FRAME if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); + f->AddImage(std::make_shared(*previous_frame_instance->GetImage())); } if (!f->has_image_data && last_final_video_frame @@ -3092,7 +3095,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } if (!f->has_image_data && last_video_frame @@ -3102,7 +3105,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } } @@ -3116,7 +3119,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = previous_frame_instance->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(previous_frame_instance->GetImage()->copy())); + f->AddImage(std::make_shared(*previous_frame_instance->GetImage())); } if (!f->has_image_data && last_final_video_frame @@ -3126,7 +3129,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = last_final_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_final_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_final_video_frame->GetImage())); } if (!f->has_image_data && last_video_frame @@ -3136,7 +3139,7 @@ void FFmpegReader::CheckWorkingFrames(int64_t requested_frame) { if (auto surface = last_video_frame->GetDeferredImage()) f->AddDeferredImage(surface); else #endif - f->AddImage(std::make_shared(last_video_frame->GetImage()->copy())); + f->AddImage(std::make_shared(*last_video_frame->GetImage())); } } diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 1c7080d6f..113f04173 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -339,4 +339,29 @@ std::vector GpuEncoders() { Logger::Instance()->Log("GPU encoders on " + gpu.name + ": " + (list.empty() ? "none" : list), Logger::LevelInfo); return encoders; } + +std::string VulkanResultName(int result) { + switch (static_cast(result)) { + case VK_SUCCESS: return "VK_SUCCESS"; + case VK_NOT_READY: return "VK_NOT_READY"; + case VK_TIMEOUT: return "VK_TIMEOUT"; + case VK_INCOMPLETE: return "VK_INCOMPLETE"; + case VK_ERROR_OUT_OF_HOST_MEMORY: return "VK_ERROR_OUT_OF_HOST_MEMORY"; + case VK_ERROR_OUT_OF_DEVICE_MEMORY: return "VK_ERROR_OUT_OF_DEVICE_MEMORY"; + case VK_ERROR_INITIALIZATION_FAILED: return "VK_ERROR_INITIALIZATION_FAILED"; + case VK_ERROR_DEVICE_LOST: return "VK_ERROR_DEVICE_LOST"; + case VK_ERROR_MEMORY_MAP_FAILED: return "VK_ERROR_MEMORY_MAP_FAILED"; + case VK_ERROR_LAYER_NOT_PRESENT: return "VK_ERROR_LAYER_NOT_PRESENT"; + case VK_ERROR_EXTENSION_NOT_PRESENT: return "VK_ERROR_EXTENSION_NOT_PRESENT"; + case VK_ERROR_FEATURE_NOT_PRESENT: return "VK_ERROR_FEATURE_NOT_PRESENT"; + case VK_ERROR_INCOMPATIBLE_DRIVER: return "VK_ERROR_INCOMPATIBLE_DRIVER"; + case VK_ERROR_TOO_MANY_OBJECTS: return "VK_ERROR_TOO_MANY_OBJECTS"; + case VK_ERROR_FORMAT_NOT_SUPPORTED: return "VK_ERROR_FORMAT_NOT_SUPPORTED"; + case VK_ERROR_FRAGMENTED_POOL: return "VK_ERROR_FRAGMENTED_POOL"; + case VK_ERROR_OUT_OF_POOL_MEMORY: return "VK_ERROR_OUT_OF_POOL_MEMORY"; + case VK_ERROR_INVALID_EXTERNAL_HANDLE: return "VK_ERROR_INVALID_EXTERNAL_HANDLE"; + case VK_ERROR_FRAGMENTATION: return "VK_ERROR_FRAGMENTATION"; + default: return "VkResult " + std::to_string(result); + } +} } diff --git a/src/GpuDevices.h b/src/GpuDevices.h index 94922d9e9..bec45442b 100644 --- a/src/GpuDevices.h +++ b/src/GpuDevices.h @@ -37,6 +37,8 @@ std::vector GpuEncoders(); /// best GPU (OPENSHOT_VULKAN_DEVICE, an index or a name, overrides it). Returns a new /// reference, or a negative FFmpeg error when there is no usable GPU. int CreateSharedVulkanDevice(AVBufferRef** output); +/// A VkResult's name (e.g. "VK_ERROR_DEVICE_LOST"), for error messages. +std::string VulkanResultName(int result); #endif } #endif diff --git a/src/GpuMetrics.cpp b/src/GpuMetrics.cpp index b000cd09e..965f8a72b 100644 --- a/src/GpuMetrics.cpp +++ b/src/GpuMetrics.cpp @@ -18,7 +18,7 @@ std::shared_ptr sessions[2] = {std::make_shared(), std::make thread_local Scope* current = nullptr; thread_local const char* reason = "Other"; const char* names[] = {"decode_gpu", "decode_cpu", "image_sources", "render_gpu", "render_cpu", - "readbacks", "uploads", "encode_gpu", "encode_cpu"}; + "readbacks", "uploads", "encode_gpu", "encode_cpu", "upload_failures"}; const char* reasons[] = {"Effect", "Transform", "Composite", "Encoder", "Other"}; } Scope::Scope(bool exporting) : previous(current), exceptions(std::uncaught_exceptions()) { diff --git a/src/GpuMetrics.h b/src/GpuMetrics.h index bcdcb1839..fddc941fb 100644 --- a/src/GpuMetrics.h +++ b/src/GpuMetrics.h @@ -11,7 +11,7 @@ void ResetGpuMetrics(bool exporting = false); #ifndef SWIG namespace gpu_metrics { enum Event { DecodeGpu, DecodeCpu, ImageSource, RenderGpu, RenderCpu, - Readback, Upload, EncodeGpu, EncodeCpu, Count }; + Readback, Upload, EncodeGpu, EncodeCpu, UploadFailure, Count }; /// Scope follows synchronous timeline work; unrelated background readers are ignored. struct Counters; class Scope { diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index c5ca42a94..5ec466506 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -4,6 +4,7 @@ #include "VideoRenderer.h" #include "../VulkanFrameImage.h" #include "../VulkanCompositor.h" +#include "../GpuDevices.h" #include "../Logger.h" #include #include @@ -35,7 +36,7 @@ extern "C" { namespace openshot { namespace { void check(VkResult result, const char* operation) { - if (result != VK_SUCCESS) throw std::runtime_error(operation); + if (result != VK_SUCCESS) throw std::runtime_error(std::string(operation) + ": " + VulkanResultName(result)); } template T glFunction(QOpenGLContext* context, const char* name) { auto function = reinterpret_cast(context->getProcAddress(name)); diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 4c8fa6016..a1eabb57b 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "VulkanCompositor.h" #include "VulkanEncodeConverter.h" +#include "GpuDevices.h" #include "VulkanFrameImage.h" #ifdef __APPLE__ #include "MacVulkanInterop.h" @@ -25,7 +26,7 @@ extern "C" { namespace openshot { namespace { void VkCheck(VkResult r, const char* operation) { - if(r != VK_SUCCESS) throw std::runtime_error(std::string(operation)+": Vulkan error "+std::to_string(r)); + if(r != VK_SUCCESS) throw std::runtime_error(std::string(operation)+": "+VulkanResultName(r)); } void AvCheck(int r, const char* operation) { if(r < 0) { diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 628d122b6..abdc0e4ec 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -47,8 +47,13 @@ void Validate(const AVFrame* frame) { throw std::invalid_argument("Expected an FFmpeg Vulkan hardware frame"); } } -namespace { std::atomic readback_count{0}; } +namespace { std::atomic readback_count{0}, upload_failures{0}; } uint64_t VulkanFrameReadbackCount() { return readback_count.load(std::memory_order_relaxed); } +uint64_t VulkanUploadFailureCount() { return upload_failures.load(std::memory_order_relaxed); } +void RecordVulkanUploadFailure() { + upload_failures.fetch_add(1, std::memory_order_relaxed); + gpu_metrics::Record(gpu_metrics::UploadFailure); +} bool VulkanRenderingEnabled() { const char* backend = std::getenv("OPENSHOT_RENDER_BACKEND"); return !(backend && std::strcmp(backend, "cpu") == 0); @@ -148,28 +153,27 @@ std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* f return RenderVulkanImage(source, nullptr, width, height, matrix, 1.0, Qt::transparent, high_quality); } namespace { +struct BufferDeleter { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; +using OwnedBuffer = std::unique_ptr; struct UploadCache { struct Entry { std::shared_ptr surface; int64_t bytes; uint64_t age; }; + struct Pool { OwnedBuffer frames; uint64_t age; }; std::mutex mutex; std::map entries; + // One frame pool per image size. A pool per upload leaked its device + // memory on NVIDIA (about 1.5 MiB per 600x600 upload) and was slow. + std::map, Pool> pools; int64_t bytes = 0; uint64_t clock = 0; }; // Never destroyed: freeing cached Vulkan frames from a static destructor at // process exit crashes the Windows NVIDIA driver, which may already be unloading. UploadCache& Uploads() { static auto* cache = new UploadCache; return *cache; } -struct BufferDeleter { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; -using OwnedBuffer = std::unique_ptr; -} -std::shared_ptr UploadVulkanImage(const QImage& image) { - if (image.isNull()) throw std::invalid_argument("Cannot upload an empty image"); - auto& cache = Uploads(); - const std::lock_guard lock(cache.mutex); - const auto key = image.cacheKey(); - auto found = cache.entries.find(key); - if (found != cache.entries.end()) { +AVBufferRef* UploadPool(UploadCache& cache, int width, int height) { + auto found = cache.pools.find({width, height}); + if (found != cache.pools.end()) { found->second.age = ++cache.clock; - return found->second.surface; + return found->second.frames.get(); } AVBufferRef* device_ptr = nullptr; Check(CreateSharedVulkanDevice(&device_ptr), "Create image upload device"); @@ -179,19 +183,51 @@ std::shared_ptr UploadVulkanImage(const QImage& image) { auto* context = reinterpret_cast(pool->data); context->format = AV_PIX_FMT_VULKAN; context->sw_format = AV_PIX_FMT_RGBA; - context->width = image.width(); context->height = image.height(); + context->width = width; context->height = height; auto* vk = reinterpret_cast(context->hwctx); vk->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); Check(av_hwframe_ctx_init(pool.get()), "Initialize image upload pool"); + // Images still in use keep their own pool reference after it is dropped here. + if (cache.pools.size() >= 8) + cache.pools.erase(std::min_element(cache.pools.begin(), cache.pools.end(), + [](const auto& a, const auto& b) { return a.second.age < b.second.age; })); + auto* frames = pool.get(); + cache.pools.emplace(std::make_pair(width, height), UploadCache::Pool{std::move(pool), ++cache.clock}); + return frames; +} +} +std::shared_ptr UploadVulkanImage(const QImage& image) { + if (image.isNull()) throw std::invalid_argument("Cannot upload an empty image"); + auto& cache = Uploads(); + const std::lock_guard lock(cache.mutex); + const auto key = image.cacheKey(); + auto found = cache.entries.find(key); + if (found != cache.entries.end()) { + found->second.age = ++cache.clock; + return found->second.surface; + } + auto* pool = UploadPool(cache, image.width(), image.height()); QImage rgba = image.convertToFormat(QImage::Format_RGBA8888_Premultiplied); auto cpu = AllocateFrame(); auto gpu = AllocateFrame(); cpu->format = AV_PIX_FMT_RGBA; cpu->width = rgba.width(); cpu->height = rgba.height(); cpu->data[0] = const_cast(rgba.constBits()); cpu->linesize[0] = rgba.bytesPerLine(); - Check(av_hwframe_get_buffer(pool.get(), gpu.get(), 0), "Allocate uploaded image"); + Check(av_hwframe_get_buffer(pool, gpu.get(), 0), "Allocate uploaded image"); Check(av_hwframe_transfer_data(gpu.get(), cpu.get(), 0), "Upload CPU image"); + // FFmpeg may copy straight from the CPU pixels (host-mapped memory) and + // returns before the copy runs; wait so rgba is not freed or reused first. + auto* uploaded = reinterpret_cast(gpu->data[0]); + auto* frames = reinterpret_cast(pool->data); + auto* device = reinterpret_cast(frames->device_ctx->hwctx); + VkSemaphoreWaitInfo copied{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + copied.semaphoreCount = 1; + copied.pSemaphores = &uploaded->sem[0]; + copied.pValues = &uploaded->sem_value[0]; + const VkResult waited = vkWaitSemaphores(device->act_dev, &copied, UINT64_MAX); + if (waited != VK_SUCCESS) + throw std::runtime_error("Wait for CPU image upload: " + VulkanResultName(waited)); gpu_metrics::Record(gpu_metrics::Upload); std::shared_ptr surface = std::make_shared(gpu.get(), true); #ifdef __APPLE__ @@ -218,6 +254,7 @@ void ReleaseVulkanUploads() { auto& cache = Uploads(); const std::lock_guard lock(cache.mutex); cache.entries.clear(); cache.bytes = 0; + cache.pools.clear(); } } diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h index 85255d55c..2ef17590b 100644 --- a/src/VulkanFrameImage.h +++ b/src/VulkanFrameImage.h @@ -11,6 +11,11 @@ namespace openshot { bool VulkanRenderingEnabled(); /// Process-wide count of GPU images materialized as CPU pixels (diagnostics). uint64_t VulkanFrameReadbackCount(); +/// Process-wide count of CPU images that could not be uploaded to the GPU. +uint64_t VulkanUploadFailureCount(); +#ifndef SWIG +void RecordVulkanUploadFailure(); +#endif /// A retained FFmpeg Vulkan image. CPU pixels are created only by Materialize(). class VulkanFrameImage final : public FrameImage { public: diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index f076e129f..8aed37199 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -627,6 +627,33 @@ TEST_CASE("PNG clips upload once and keep animated transforms on Vulkan", "[vulk timeline.Close(); } +TEST_CASE("FFmpeg still images and cover art upload once", "[vulkan][hardware][upload]") { + if (!VulkanAvailable()) return; + QTemporaryDir temporary; + REQUIRE(temporary.isValid()); + const auto path = temporary.filePath("cover.jpg"); + QImage bitmap(64, 64, QImage::Format_RGB32); + bitmap.fill(QColor(255, 0, 0)); + REQUIRE(bitmap.save(path)); + // FFmpegReader fills each frame of a still image from the previous one, as + // it does for an audio file's cover art; that must not upload every frame. + openshot::FFmpegReader reader(path.toStdString()); + REQUIRE(reader.info.has_single_image); + openshot::Clip clip(&reader); + openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + openshot::ResetGpuMetrics(); + for (int i = 1; i <= 24; ++i) + REQUIRE(timeline.GetFrame(i)->GetDeferredImage()); + Json::Reader parser; + Json::Value metrics; + REQUIRE(parser.parse(openshot::GpuMetricsSnapshot(), metrics)); + CHECK(metrics["uploads"].asUInt64() <= 1); + CHECK(metrics["upload_failures"].asUInt64() == 0); + timeline.Close(); +} + TEST_CASE("Saved GPU preferences reach the native runtime environment", "[vulkan][settings]") { struct Restore { std::string backend, device; From 5b58759b894d351b22d3377997c11976a0e00448 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 1 Nov 2026 23:17:07 -0600 Subject: [PATCH 30/50] Recover the GPU preview from errors and continue on the CPU after device loss A failed GPU preview draw now frees cached GPU memory and retries after a growing pause (0.5 s doubling to 30 s) instead of failing on every frame. VK_ERROR_DEVICE_LOST marks the device lost for the session: Vulkan rendering is disabled and open readers reopen with CPU decoding, so editing continues without a restart. --- src/FFmpegReader.cpp | 8 ++++++ src/GpuDevices.cpp | 9 +++++++ src/GpuDevices.h | 5 ++++ src/Qt/VulkanOpenGLPresenter.cpp | 32 +++++++++++++++++++++-- src/VulkanCompositor.cpp | 4 ++- src/VulkanFrameImage.cpp | 3 ++- tests/CMakeLists.txt | 2 +- tests/VulkanDeviceLost.cpp | 44 ++++++++++++++++++++++++++++++++ 8 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 tests/VulkanDeviceLost.cpp diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 66e57bd9f..9a52d0842 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -1328,6 +1328,14 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { // Prevent async calls to the remainder of this code const std::lock_guard lock(getFrameMutex); +#if USE_HW_ACCEL + // A lost GPU device cannot decode; reopen this reader on the CPU. + if (hw_de_on && !force_sw_decode && VulkanDeviceLost()) { + hw_decode_failed = true; + ReopenWithoutHardwareDecode(requested_frame); + } +#endif + // Check the cache a 2nd time (due to the potential previous lock) frame = final_cache.GetFrame(requested_frame); if (frame) { diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 113f04173..3c7c21d96 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -4,6 +4,7 @@ #include "Logger.h" #include "VulkanFrameImage.h" #include +#include #include #include #include @@ -340,6 +341,14 @@ std::vector GpuEncoders() { return encoders; } +namespace { std::atomic device_lost{false}; } +bool VulkanDeviceLost() { return device_lost.load(std::memory_order_relaxed); } +void ReportVulkanResult(int result) { + if (result != VK_ERROR_DEVICE_LOST || device_lost.exchange(true)) return; + Logger::Instance()->Log("GPU device lost (driver reset or GPU timeout); using the CPU for the rest of this session", + Logger::LevelWarning); +} + std::string VulkanResultName(int result) { switch (static_cast(result)) { case VK_SUCCESS: return "VK_SUCCESS"; diff --git a/src/GpuDevices.h b/src/GpuDevices.h index bec45442b..9a9ee55cb 100644 --- a/src/GpuDevices.h +++ b/src/GpuDevices.h @@ -32,6 +32,9 @@ GpuDevice ActiveGpuDevice(); /// Hardware encoders (e.g. "h264_nvenc", "hevc_vaapi") that open on the active /// card. Tested once per process with a tiny frame size; takes a fraction of a second. std::vector GpuEncoders(); +/// True after a Vulkan call reported VK_ERROR_DEVICE_LOST (a driver reset or GPU +/// timeout). Processing then stays on the CPU for the rest of the session. +bool VulkanDeviceLost(); #ifndef SWIG /// The shared Vulkan device every reader decodes on, created on first use with the /// best GPU (OPENSHOT_VULKAN_DEVICE, an index or a name, overrides it). Returns a new @@ -39,6 +42,8 @@ std::vector GpuEncoders(); int CreateSharedVulkanDevice(AVBufferRef** output); /// A VkResult's name (e.g. "VK_ERROR_DEVICE_LOST"), for error messages. std::string VulkanResultName(int result); +/// Record a failed Vulkan call; VK_ERROR_DEVICE_LOST sets VulkanDeviceLost(). +void ReportVulkanResult(int result); #endif } #endif diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 5ec466506..aaf8f889c 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -29,6 +29,7 @@ #include #include #include +#include extern "C" { #include } @@ -36,7 +37,9 @@ extern "C" { namespace openshot { namespace { void check(VkResult result, const char* operation) { - if (result != VK_SUCCESS) throw std::runtime_error(std::string(operation) + ": " + VulkanResultName(result)); + if (result == VK_SUCCESS) return; + ReportVulkanResult(result); + throw std::runtime_error(std::string(operation) + ": " + VulkanResultName(result)); } template T glFunction(QOpenGLContext* context, const char* name) { auto function = reinterpret_cast(context->getProcAddress(name)); @@ -98,6 +101,11 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { uint64_t native_draws = 0; uint64_t fallback_frames = 0; std::string last_failure; + // After a failed draw the CPU shows frames until retry_after, and each + // further failure in a row doubles the wait (up to 30 seconds). + int failures_in_a_row = 0; + std::chrono::steady_clock::time_point retry_after; + std::string paused_reason; QPointer logged_context; bool identity_logged = false; SharedTarget target; @@ -371,6 +379,9 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { return unavailable("Vulkan presentation requires desktop OpenGL; Qt selected OpenGL ES/ANGLE (try QT_OPENGL=desktop)"); if (!frame || !widget) return unavailable("No video frame or preview widget"); if (fallback_delivered) return false; + if (VulkanDeviceLost()) + return unavailable("GPU device lost (driver reset or GPU timeout); the CPU is used for the rest of this session"); + if (std::chrono::steady_clock::now() < retry_after) return unavailable(paused_reason); auto authoritative = frame->GetDeferredImage(); if (!authoritative) return unavailable("Frame has no deferred GPU surface"); auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(authoritative); @@ -580,10 +591,27 @@ VulkanOpenGLPresenter::VulkanOpenGLPresenter(uintptr_t widget) VulkanOpenGLPresenter::~VulkanOpenGLPresenter() = default; bool VulkanOpenGLPresenter::Draw(int x, int y, int width, int height) { try { - if (impl->draw(x, y, width, height)) { impl->last_failure.clear(); ++impl->native_draws; return true; } + if (impl->draw(x, y, width, height)) { + impl->last_failure.clear(); + impl->failures_in_a_row = 0; + ++impl->native_draws; + return true; + } } catch (const std::exception& error) { impl->last_failure = error.what(); qWarning() << "Vulkan presentation fallback:" << error.what(); + if (!VulkanDeviceLost()) { + // Usually low GPU memory: free cached GPU images, then retry later + // instead of failing again on every frame. + ReleaseVulkanMemory(); + const int failures = ++impl->failures_in_a_row; + const auto delay = std::chrono::milliseconds(std::min(30000, 500 << std::min(failures - 1, 6))); + impl->retry_after = std::chrono::steady_clock::now() + delay; + impl->paused_reason = std::string("GPU preview paused after an error; retrying shortly (") + + error.what() + ")"; + Logger::Instance()->Log("GPU preview failed (" + std::to_string(failures) + " in a row); retrying in " + + std::to_string(delay.count()) + " ms: " + error.what(), Logger::LevelWarning); + } } impl->fallback(); return false; diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index a1eabb57b..5ec776f6c 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -26,7 +26,9 @@ extern "C" { namespace openshot { namespace { void VkCheck(VkResult r, const char* operation) { - if(r != VK_SUCCESS) throw std::runtime_error(std::string(operation)+": "+VulkanResultName(r)); + if(r == VK_SUCCESS) return; + ReportVulkanResult(r); + throw std::runtime_error(std::string(operation)+": "+VulkanResultName(r)); } void AvCheck(int r, const char* operation) { if(r < 0) { diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index abdc0e4ec..1f668438c 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -56,7 +56,7 @@ void RecordVulkanUploadFailure() { } bool VulkanRenderingEnabled() { const char* backend = std::getenv("OPENSHOT_RENDER_BACKEND"); - return !(backend && std::strcmp(backend, "cpu") == 0); + return !(backend && std::strcmp(backend, "cpu") == 0) && !VulkanDeviceLost(); } struct VulkanFrameImage::Impl { OwnedFrame frame; @@ -226,6 +226,7 @@ std::shared_ptr UploadVulkanImage(const QImage& image) { copied.pSemaphores = &uploaded->sem[0]; copied.pValues = &uploaded->sem_value[0]; const VkResult waited = vkWaitSemaphores(device->act_dev, &copied, UINT64_MAX); + ReportVulkanResult(waited); if (waited != VK_SUCCESS) throw std::runtime_error("Wait for CPU image upload: " + VulkanResultName(waited)); gpu_metrics::Record(gpu_metrics::Upload); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 76313ae03..eec8019f0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -100,7 +100,7 @@ set(OPENSHOT_TESTS ) if(ENABLE_GPU_FRAME) - list(APPEND OPENSHOT_TESTS VulkanCompositor VulkanPresentation) + list(APPEND OPENSHOT_TESTS VulkanCompositor VulkanPresentation VulkanDeviceLost) endif() # The SIGPIPE crash-handler test is POSIX-only. Do not create an empty diff --git a/tests/VulkanDeviceLost.cpp b/tests/VulkanDeviceLost.cpp new file mode 100644 index 000000000..11368e95e --- /dev/null +++ b/tests/VulkanDeviceLost.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +// A lost device is permanent for the process, so this test has its own binary. +#include "openshot_catch.h" +#include "FFmpegReader.h" +#include "GpuDevices.h" +#include "VulkanFrameImage.h" +extern "C" { +#include +#include +} + +TEST_CASE("A lost GPU device moves open readers and new work to the CPU", "[vulkan][recovery]") { + if (!openshot::VulkanRenderingEnabled()) { + WARN("Skipping Vulkan test: OPENSHOT_RENDER_BACKEND=cpu"); + return; + } + AVBufferRef* device = nullptr; + if (openshot::CreateSharedVulkanDevice(&device) < 0) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return; + } + av_buffer_unref(&device); + CHECK(openshot::VulkanResultName(VK_ERROR_DEVICE_LOST) == "VK_ERROR_DEVICE_LOST"); + openshot::FFmpegReader reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + reader.Open(); + const bool gpu_decoded = reader.GetFrame(1)->GetDeferredImage() != nullptr; + + // Other errors (such as low GPU memory) are recoverable. + openshot::ReportVulkanResult(VK_ERROR_OUT_OF_DEVICE_MEMORY); + CHECK_FALSE(openshot::VulkanDeviceLost()); + CHECK(openshot::VulkanRenderingEnabled()); + + openshot::ReportVulkanResult(VK_ERROR_DEVICE_LOST); + CHECK(openshot::VulkanDeviceLost()); + CHECK_FALSE(openshot::VulkanRenderingEnabled()); + // The open reader reopens with CPU decoding instead of using the lost device. + auto frame = reader.GetFrame(48); + REQUIRE(frame); + CHECK_FALSE(frame->GetDeferredImage()); + CHECK(frame->GetImage()->width() == 1280); + if (!gpu_decoded) WARN("GPU decode was not active before the loss on this system"); + reader.Close(); +} From 8fcaa3bc97714fa30209cd3b5b6a24ca2bea1958 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 2 Nov 2026 11:47:20 -0600 Subject: [PATCH 31/50] Add temporary GPU debug logging and preview handoff switches OPENSHOT_GPU_DEBUG=1 logs centre pixels at each preview handoff step, the shared buffer setup, OpenGL import errors, and draw geometry, and saves a crash dump on Windows. OPENSHOT_PRESENTATION_TEST=pattern, OPENSHOT_PRESENTATION_SYNC=wait and OPENSHOT_PRESENTATION_DEDICATED=0|1 change one part of the handoff each. Vulkan readback failures now log the frame and device details. Search for GpuDebug to remove. --- src/CrashHandler.cpp | 18 ++++ src/GpuDebug.h | 30 ++++++ src/Qt/VulkanOpenGLPresenter.cpp | 176 +++++++++++++++++++++++++++++-- src/VulkanFrameImage.cpp | 30 +++++- 4 files changed, 243 insertions(+), 11 deletions(-) create mode 100644 src/GpuDebug.h diff --git a/src/CrashHandler.cpp b/src/CrashHandler.cpp index 153e8ab66..d009d0529 100644 --- a/src/CrashHandler.cpp +++ b/src/CrashHandler.cpp @@ -55,6 +55,23 @@ CrashHandler *CrashHandler::Instance() } #ifdef __MINGW32__ +// GpuDebug: with OPENSHOT_GPU_DEBUG set, save a small crash dump (thread +// stacks only) to %USERPROFILE%\.openshot_qt for debugging test builds. +static void writeMiniDump() +{ + const char* profile = getenv("USERPROFILE"); + if (!getenv("OPENSHOT_GPU_DEBUG") || !profile) + return; + char path[MAX_PATH]; + snprintf(path, sizeof(path), "%s\\.openshot_qt\\openshot-crash-%lu.dmp", profile, GetCurrentProcessId()); + HANDLE file = CreateFileA(path, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) + return; + if (MiniDumpWriteDump(GetCurrentProcess(), GetCurrentProcessId(), file, MiniDumpNormal, NULL, NULL, NULL)) + fprintf(stderr, "Crash dump saved: %s\n", path); + CloseHandle(file); +} + // Windows exception handler void CrashHandler::abortHandler(int signum) { @@ -76,6 +93,7 @@ void CrashHandler::abortHandler(int signum) // Dump a stack trace. printStackTrace(stderr, 63); + writeMiniDump(); // Quit exit( signum ); diff --git a/src/GpuDebug.h b/src/GpuDebug.h new file mode 100644 index 000000000..e71383ff8 --- /dev/null +++ b/src/GpuDebug.h @@ -0,0 +1,30 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later + +// Temporary switches for debugging GPU test builds; search for "GpuDebug" +// to find (and later remove) every use. +// OPENSHOT_GPU_DEBUG=1 extra logging only, no behavior change +// OPENSHOT_PRESENTATION_TEST=pattern preview shows solid magenta instead of video +// OPENSHOT_PRESENTATION_SYNC=wait CPU waits instead of Vulkan/OpenGL semaphores +// OPENSHOT_PRESENTATION_DEDICATED=0|1 force the preview buffer's memory allocation style +#ifndef OPENSHOT_GPU_DEBUG_H +#define OPENSHOT_GPU_DEBUG_H + +#include + +namespace openshot { + +// OPENSHOT_TRACE_PRESENTATION is the older name for the same logging. +inline bool GpuDebug() { + static const bool enabled = qEnvironmentVariableIsSet("OPENSHOT_GPU_DEBUG") || + qEnvironmentVariableIsSet("OPENSHOT_TRACE_PRESENTATION"); + return enabled; +} + +inline QString GpuDebugSetting(const char* name) { + return qEnvironmentVariable(name).trimmed().toLower(); +} + +} + +#endif diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index aaf8f889c..05fe84499 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -5,6 +5,7 @@ #include "../VulkanFrameImage.h" #include "../VulkanCompositor.h" #include "../GpuDevices.h" +#include "../GpuDebug.h" #include "../Logger.h" #include #include @@ -30,6 +31,7 @@ #include #include #include +#include extern "C" { #include } @@ -64,6 +66,11 @@ using ImportMemory = PFNGLIMPORTMEMORYFDEXTPROC; using ImportSemaphore = PFNGLIMPORTSEMAPHOREFDEXTPROC; #endif constexpr VkImageSubresourceRange kColor = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; +std::string pixelText(const unsigned char* pixel) { + char text[16]; + std::snprintf(text, sizeof(text), "#%02x%02x%02x%02x", pixel[0], pixel[1], pixel[2], pixel[3]); + return text; +} } // Frames reach OpenGL through a presenter-owned linear buffer, never through // shared image memory. NVIDIA's Windows driver returns zeros when FFmpeg or @@ -92,6 +99,10 @@ struct SharedTarget { std::unique_ptr program; PFNGLWAITSEMAPHOREEXTPROC wait_semaphore = nullptr; PFNGLSIGNALSEMAPHOREEXTPROC signal_semaphore = nullptr; + // GpuDebug: a host-visible copy of the frame's centre pixel. + VkBuffer probe = VK_NULL_HANDLE; + VkDeviceMemory probe_memory = VK_NULL_HANDLE; + const unsigned char* probe_pixel = nullptr; }; struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { QPointer widget; @@ -108,6 +119,8 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { std::string paused_reason; QPointer logged_context; bool identity_logged = false; + uint64_t debug_frames = 0; + std::string debug_geometry; SharedTarget target; bool unavailable(const std::string& reason) { last_failure = reason; return false; } explicit Impl(QWidget* value) : widget(value) { @@ -173,10 +186,34 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (semaphore) vkDestroySemaphore(target.device, semaphore, nullptr); if (target.pixels) vkDestroyBuffer(target.device, target.pixels, nullptr); if (target.memory) vkFreeMemory(target.device, target.memory, nullptr); + if (target.probe) vkDestroyBuffer(target.device, target.probe, nullptr); + if (target.probe_memory) vkFreeMemory(target.device, target.probe_memory, nullptr); } av_buffer_unref(&target.device_ref); target = SharedTarget(); } + // GpuDebug: a 4-byte host-visible buffer that receives the frame's centre pixel. + void createProbe(const VkPhysicalDeviceMemoryProperties& memoryProperties) { + VkBufferCreateInfo create{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + create.size = 4; + create.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + check(vkCreateBuffer(target.device, &create, nullptr, &target.probe), "Create debug probe"); + VkMemoryRequirements needs; + vkGetBufferMemoryRequirements(target.device, target.probe, &needs); + const VkMemoryPropertyFlags wanted = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocate.allocationSize = needs.size; + allocate.memoryTypeIndex = UINT32_MAX; + for (uint32_t i = 0; i < memoryProperties.memoryTypeCount && allocate.memoryTypeIndex == UINT32_MAX; ++i) + if ((needs.memoryTypeBits & (1u << i)) && (memoryProperties.memoryTypes[i].propertyFlags & wanted) == wanted) + allocate.memoryTypeIndex = i; + if (allocate.memoryTypeIndex == UINT32_MAX) throw std::runtime_error("No host-visible debug probe memory"); + check(vkAllocateMemory(target.device, &allocate, nullptr, &target.probe_memory), "Allocate debug probe"); + check(vkBindBufferMemory(target.device, target.probe, target.probe_memory, 0), "Bind debug probe"); + void* mapped = nullptr; + check(vkMapMemory(target.device, target.probe_memory, 0, VK_WHOLE_SIZE, 0, &mapped), "Map debug probe"); + target.probe_pixel = static_cast(mapped); + } void createTarget(QOpenGLContext* context, const AVHWFramesContext* fc, int width, int height) { auto vk = static_cast(fc->device_ctx->hwctx); target.device_ref = av_buffer_ref(fc->device_ref); @@ -232,7 +269,13 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { (memoryProperties.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) memoryType = i; if (memoryType == UINT32_MAX) throw std::runtime_error("No device-local presentation memory"); - const bool useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; + bool useDedicated = dedicated.prefersDedicatedAllocation || dedicated.requiresDedicatedAllocation; + // GpuDebug: OPENSHOT_PRESENTATION_DEDICATED=0|1 overrides the driver's preference. + const auto forceDedicated = GpuDebugSetting("OPENSHOT_PRESENTATION_DEDICATED"); + if (forceDedicated == "1") + useDedicated = true; + else if (forceDedicated == "0" && !dedicated.requiresDedicatedAllocation) + useDedicated = false; VkMemoryDedicatedAllocateInfo dedicatedInfo{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO}; dedicatedInfo.buffer = target.pixels; VkExportMemoryAllocateInfo exportInfo{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO, useDedicated ? &dedicatedInfo : nullptr}; @@ -242,10 +285,33 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { allocate.memoryTypeIndex = memoryType; check(vkAllocateMemory(target.device, &allocate, nullptr, &target.memory), "Allocate presentation memory"); check(vkBindBufferMemory(target.device, target.pixels, target.memory, 0), "Bind presentation memory"); + if (GpuDebug()) { + const auto& type = memoryProperties.memoryTypes[memoryType]; + char flags[64]; + std::snprintf(flags, sizeof(flags), "0x%x; heap %u flags=0x%x", type.propertyFlags, type.heapIndex, + memoryProperties.memoryHeaps[type.heapIndex].flags); + Logger::Instance()->Log("GPU debug presentation buffer: " + std::to_string(width) + "x" + + std::to_string(height) + "; size=" + std::to_string(size) + "; allocation=" + + std::to_string(requirements.memoryRequirements.size) + "; memory type " + std::to_string(memoryType) + + " flags=" + flags + "; dedicated prefers=" + std::to_string(dedicated.prefersDedicatedAllocation) + + " requires=" + std::to_string(dedicated.requiresDedicatedAllocation) + " used=" + + std::to_string(useDedicated) + "; override=" + forceDedicated.toStdString(), Logger::LevelInfo); + createProbe(memoryProperties); + } + // GpuDebug: check OpenGL errors after each import step, not only at the end. + GLenum importError = GL_NO_ERROR; + const auto glStep = [&](const char* name) { + if (!GpuDebug()) return; + const GLenum error = context->functions()->glGetError(); + if (importError == GL_NO_ERROR) importError = error; + Logger::Instance()->Log(std::string("GPU debug OpenGL ") + name + ": error=" + std::to_string(error), + Logger::LevelInfo); + }; createMemory(1, &target.gl_memory); GLint glDedicated = useDedicated; memoryParameter(target.gl_memory, GL_DEDICATED_MEMORY_OBJECT_EXT, &glDedicated); + glStep("create memory object"); #ifdef _WIN32 VkMemoryGetWin32HandleInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR}; memoryInfo.memory = target.memory; @@ -255,6 +321,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryHandle); // Win32 imports retain the resource, not ownership of the handle. CloseHandle(memoryHandle); + glStep("import memory"); #else VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; memoryInfo.memory = target.memory; @@ -262,6 +329,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { int memoryFd = -1; check(getMemoryHandle(target.device, &memoryInfo, &memoryFd), "Export presentation memory"); importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryFd); // GL owns the fd. + glStep("import memory"); #endif GLint previousUnpack = 0, previousTexture = 0; gl->glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previousUnpack); @@ -269,6 +337,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { gl->glGenBuffers(1, &target.gl_pixels); gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, target.gl_pixels); bufferMemory(GL_PIXEL_UNPACK_BUFFER, size, target.gl_memory, 0); + glStep("buffer storage"); gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, previousUnpack); gl->glGenTextures(1, &target.texture); gl->glBindTexture(GL_TEXTURE_2D, target.texture); @@ -277,6 +346,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); gl->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); + glStep("texture"); gl->glBindTexture(GL_TEXTURE_2D, previousTexture); VkExportSemaphoreCreateInfo exportSemaphore{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO}; @@ -301,8 +371,10 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { check(getSemaphoreHandle(target.device, &handleInfo, &semaphoreFd), "Export presentation semaphore"); importSemaphore(target.gl_semaphores[direction], kGLHandle, semaphoreFd); #endif + glStep("import semaphore"); } - if (gl->glGetError() != GL_NO_ERROR) throw std::runtime_error("OpenGL external memory import failed"); + if (importError != GL_NO_ERROR || gl->glGetError() != GL_NO_ERROR) + throw std::runtime_error("OpenGL external memory import failed"); VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; @@ -352,7 +424,11 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (qEnvironmentVariable("OPENSHOT_PRESENTATION_BACKEND") == "cpu") return unavailable("OPENSHOT_PRESENTATION_BACKEND=cpu (Vulkan processing remains enabled)"); auto lastStage = std::chrono::steady_clock::now(); - const bool trace = qEnvironmentVariableIsSet("OPENSHOT_TRACE_PRESENTATION"); + const bool trace = GpuDebug(); + // GpuDebug switches: draw solid magenta instead of the video, or have + // the CPU wait for each side instead of sharing semaphores. + static const bool pattern = GpuDebugSetting("OPENSHOT_PRESENTATION_TEST") == "pattern"; + static const bool cpuWait = GpuDebugSetting("OPENSHOT_PRESENTATION_SYNC") == "wait"; auto stage = [&](const char* name) { if (!trace) return; auto now = std::chrono::steady_clock::now(); @@ -443,6 +519,9 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { bool locked = false, copySubmitted = false; uint64_t copyValue = 0; + // GpuDebug: centre pixel at each step, for the first 10 frames and every 60th. + const bool probeFrame = trace && (++debug_frames <= 10 || debug_frames % 60 == 0); + std::string pixels; try { if (target.context != context || target.device != vk->act_dev || target.width != source->width || target.height != source->height) { @@ -480,7 +559,22 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { VkBufferImageCopy region{}; region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; region.imageExtent = {static_cast(target.width), static_cast(target.height), 1}; - vkCmdCopyImageToBuffer(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.pixels, 1, ®ion); + if (pattern) + vkCmdFillBuffer(copy, target.pixels, 0, VK_WHOLE_SIZE, 0xFFFF00FFu); // RGBA ff00ffff + else + vkCmdCopyImageToBuffer(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.pixels, 1, ®ion); + if (target.probe) { + VkBufferImageCopy centre{}; + centre.imageSubresource = region.imageSubresource; + centre.imageOffset = {target.width / 2, target.height / 2, 0}; + centre.imageExtent = {1, 1, 1}; + vkCmdCopyImageToBuffer(copy, image->img[0], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, target.probe, 1, ¢re); + VkMemoryBarrier toHost{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + toHost.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toHost.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(copy, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, + 0, 1, &toHost, 0, nullptr, 0, nullptr); + } VkBufferMemoryBarrier release{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; release.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; release.srcQueueFamilyIndex = target.family; @@ -497,7 +591,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { VkTimelineSemaphoreSubmitInfo timeline{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; timeline.waitSemaphoreValueCount = 1; timeline.pWaitSemaphoreValues = &waitValue; - timeline.signalSemaphoreValueCount = 2; + timeline.signalSemaphoreValueCount = cpuWait ? 1 : 2; timeline.pSignalSemaphoreValues = signalValues; const VkPipelineStageFlags waitStage = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; VkSubmitInfo copyInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO, &timeline}; @@ -506,7 +600,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { copyInfo.pWaitDstStageMask = &waitStage; copyInfo.commandBufferCount = 1; copyInfo.pCommandBuffers = © - copyInfo.signalSemaphoreCount = 2; + copyInfo.signalSemaphoreCount = cpuWait ? 1 : 2; copyInfo.pSignalSemaphores = copySignals; submit(copyInfo, VK_NULL_HANDLE); copySubmitted = true; @@ -516,8 +610,30 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { vfc->unlock_frame(fc, image); locked = false; stage("copy submit"); + if (cpuWait || probeFrame) { + VkSemaphoreWaitInfo copyDone{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO}; + copyDone.semaphoreCount = 1; + copyDone.pSemaphores = &image->sem[0]; + copyDone.pValues = ©Value; + check(vkWaitSemaphores(target.device, ©Done, UINT64_MAX), "Wait for presentation copy"); + if (probeFrame && target.probe_pixel) + pixels += "vulkan=" + pixelText(target.probe_pixel); + } - target.wait_semaphore(target.gl_semaphores[0], 1, &target.gl_pixels, 0, nullptr, nullptr); + if (!cpuWait) + target.wait_semaphore(target.gl_semaphores[0], 1, &target.gl_pixels, 0, nullptr, nullptr); + if (probeFrame) { + using GetBufferSubData = void (QOPENGLF_APIENTRYP)(GLenum, GLintptr, GLsizeiptr, void*); + auto getBufferSubData = reinterpret_cast(context->getProcAddress("glGetBufferSubData")); + unsigned char pixel[4] = {0, 0, 0, 0}; + GLint previousCopy = 0; + gl->glGetIntegerv(GL_COPY_READ_BUFFER_BINDING, &previousCopy); + gl->glBindBuffer(GL_COPY_READ_BUFFER, target.gl_pixels); + if (getBufferSubData) + getBufferSubData(GL_COPY_READ_BUFFER, (GLintptr(target.height / 2) * target.width + target.width / 2) * 4, 4, pixel); + gl->glBindBuffer(GL_COPY_READ_BUFFER, previousCopy); + pixels += " buffer=" + pixelText(pixel); + } GLint viewport[4], previousVertexArray = 0, previousUnpack = 0, unpackAlignment = 4, unpackRowLength = 0; gl->glGetIntegerv(GL_VIEWPORT, viewport); gl->glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previousVertexArray); @@ -531,6 +647,20 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); gl->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, target.width, target.height, GL_RGBA, GL_UNSIGNED_BYTE, nullptr); gl->glBindBuffer(GL_PIXEL_UNPACK_BUFFER, previousUnpack); + if (probeFrame) { + unsigned char pixel[4] = {0, 0, 0, 0}; + GLint previousFramebuffer = 0; + GLuint probe = 0; + gl->glGetIntegerv(GL_FRAMEBUFFER_BINDING, &previousFramebuffer); + gl->glGenFramebuffers(1, &probe); + gl->glBindFramebuffer(GL_FRAMEBUFFER, probe); + gl->glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, target.texture, 0); + gl->glReadPixels(target.width / 2, target.height / 2, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixel); + gl->glBindFramebuffer(GL_FRAMEBUFFER, previousFramebuffer); + gl->glDeleteFramebuffers(1, &probe); + const GLenum readError = gl->glGetError(); + pixels += " texture=" + pixelText(pixel) + (readError ? " (GL error " + std::to_string(readError) + ")" : ""); + } gl->glPixelStorei(GL_UNPACK_ALIGNMENT, unpackAlignment); gl->glPixelStorei(GL_UNPACK_ROW_LENGTH, unpackRowLength); gl->glViewport(x, y, width, height); @@ -552,17 +682,43 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { qWarning() << "GPU presentation draw state: error=" << error << "framebuffer=" << framebuffer << "program=" << activeProgram << "texture=" << boundTexture; + // Log where the frame lands whenever that changes. + const qreal ratio = widget ? widget->devicePixelRatioF() : 0.0; + const std::string geometry = "draw " + std::to_string(x) + "," + std::to_string(y) + " " + + std::to_string(width) + "x" + std::to_string(height) + "; frame " + std::to_string(target.width) + + "x" + std::to_string(target.height) + "; GL viewport " + std::to_string(viewport[0]) + "," + + std::to_string(viewport[1]) + " " + std::to_string(viewport[2]) + "x" + std::to_string(viewport[3]) + + "; framebuffer " + std::to_string(framebuffer) + "; widget " + + std::to_string(widget ? widget->width() : 0) + "x" + std::to_string(widget ? widget->height() : 0) + + " at scale " + std::to_string(ratio) + "; pattern=" + std::to_string(pattern) + + " cpu wait=" + std::to_string(cpuWait) + "; GL error=" + std::to_string(error); + if (geometry != debug_geometry) { + debug_geometry = geometry; + Logger::Instance()->Log("GPU debug presentation " + geometry, Logger::LevelInfo); + } + } + if (probeFrame) { + unsigned char pixel[4] = {0, 0, 0, 0}; + gl->glReadPixels(x + width / 2, y + height / 2, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixel); + const GLenum readError = gl->glGetError(); + pixels += " screen=" + pixelText(pixel) + (readError ? " (GL error " + std::to_string(readError) + ")" : ""); + Logger::Instance()->Log("GPU debug preview pixels, frame " + std::to_string(debug_frames) + ": " + pixels, + Logger::LevelInfo); } extra->glBindVertexArray(previousVertexArray); target.program->release(); gl->glViewport(viewport[0], viewport[1], viewport[2], viewport[3]); // Signals once the unpack above has finished reading the buffer. - target.signal_semaphore(target.gl_semaphores[1], 1, &target.gl_pixels, 0, nullptr, nullptr); - gl->glFlush(); + if (cpuWait) { + gl->glFinish(); + } else { + target.signal_semaphore(target.gl_semaphores[1], 1, &target.gl_pixels, 0, nullptr, nullptr); + gl->glFlush(); + } stage("GL draw flush"); VkSubmitInfo acquireInfo{VK_STRUCTURE_TYPE_SUBMIT_INFO}; - acquireInfo.waitSemaphoreCount = 1; + acquireInfo.waitSemaphoreCount = cpuWait ? 0 : 1; acquireInfo.pWaitSemaphores = &target.handoff[1]; acquireInfo.pWaitDstStageMask = &waitStage; acquireInfo.commandBufferCount = 1; diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 1f668438c..30da0c2f0 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -3,6 +3,7 @@ #include "GpuMetrics.h" #include "VulkanFrameImage.h" #include "GpuDevices.h" +#include "Logger.h" #include #include #include "VulkanCompositor.h" @@ -41,6 +42,29 @@ OwnedFrame AllocateFrame() { if (!f) throw std::bad_alloc(); return f; } +// Readback failures are rare and hard to reproduce, so log what FFmpeg was given. +void LogReadbackFailure(const AVFrame* frame, int result) { + auto fc = reinterpret_cast(frame->hw_frames_ctx->data); + auto vfc = static_cast(fc->hwctx); + auto vk = static_cast(fc->device_ctx->hwctx); + bool hostImport = false; + for (int i = 0; i < vk->nb_enabled_dev_extensions; ++i) + hostImport |= std::strcmp(vk->enabled_dev_extensions[i], VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) == 0; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT host{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &host}; + vkGetPhysicalDeviceProperties2(vk->phys_dev, &properties); + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(result, error, sizeof(error)); + const char* format = av_get_pix_fmt_name(fc->sw_format); + Logger::Instance()->Log(std::string("Vulkan readback failed: ") + error + " (" + std::to_string(result) + + "); frame " + std::to_string(frame->width) + "x" + std::to_string(frame->height) + "; pool " + + std::to_string(fc->width) + "x" + std::to_string(fc->height) + " " + (format ? format : "unknown") + + "; Vulkan format " + std::to_string(vfc->format[0]) + "; tiling " + std::to_string(vfc->tiling) + + "; usage 0x" + QString::number(vfc->usage, 16).toStdString() + "; host memory import " + + (hostImport ? "on, alignment " + std::to_string(host.minImportedHostPointerAlignment) : std::string("off")) + + "; row pitch alignment " + std::to_string(properties.properties.limits.optimalBufferCopyRowPitchAlignment) + + "; device " + properties.properties.deviceName, Logger::LevelWarning); +} void Validate(const AVFrame* frame) { if (!frame || frame->format != AV_PIX_FMT_VULKAN || !frame->hw_frames_ctx || !frame->data[0] || frame->width <= 0 || frame->height <= 0) @@ -102,7 +126,11 @@ QImage VulkanFrameImage::Materialize() const { Check(ReadMacVulkanFrame(cpu.get(), impl->frame.get()), "Read back Mac Vulkan frame"); else #endif - Check(av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0), "Read back Vulkan frame"); + { + const int result = av_hwframe_transfer_data(cpu.get(), impl->frame.get(), 0); + if (result < 0) LogReadbackFailure(impl->frame.get(), result); + Check(result, "Read back Vulkan frame"); + } readback_count.fetch_add(1,std::memory_order_relaxed); gpu_metrics::Record(gpu_metrics::Readback); Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); From fcf8dc1dc2590dde6657f234a20e943e91ff5177 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Mon, 2 Nov 2026 21:57:58 -0600 Subject: [PATCH 32/50] Keep Win32 preview handles open and log why frames are read back The RTX 2060 debug logs showed OpenGL reading zeros from the shared preview buffer while Vulkan wrote the right pixels, even with a fill pattern and CPU waits. The presenter closed each Win32 handle right after importing it; keep them open until the shared target is released. OPENSHOT_PRESENTATION_HANDLE=kmt tries global KMT handles instead. With OPENSHOT_GPU_DEBUG=1, readbacks log their reason, and thumbnails and image saves are tagged. --- src/Frame.cpp | 3 +++ src/GpuMetrics.cpp | 1 + src/GpuMetrics.h | 2 ++ src/Qt/VulkanOpenGLPresenter.cpp | 31 ++++++++++++++++++++++++------- src/VulkanFrameImage.cpp | 9 ++++++++- 5 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/Frame.cpp b/src/Frame.cpp index b47adcef6..6447326b2 100644 --- a/src/Frame.cpp +++ b/src/Frame.cpp @@ -20,6 +20,7 @@ #include "AudioBufferSource.h" #include "AudioResampler.h" #include "QtUtilities.h" +#include "GpuMetrics.h" #include #include @@ -540,6 +541,7 @@ ChannelLayout Frame::ChannelsLayout() // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG) void Frame::Save(std::string path, float scale, std::string format, int quality) { + gpu_metrics::Reason gpu_reason("Image file"); // Get preview image std::shared_ptr previewImage = GetImage(); @@ -568,6 +570,7 @@ void Frame::Save(std::string path, float scale, std::string format, int quality) // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG) void Frame::Thumbnail(std::string path, int new_width, int new_height, std::string mask_path, std::string overlay_path, std::string background_color, bool ignore_aspect, std::string format, int quality, float rotate, ScaleType scale_mode) { + gpu_metrics::Reason gpu_reason("Thumbnail"); // Create blank thumbnail image & fill background color auto thumbnail = std::make_shared( diff --git a/src/GpuMetrics.cpp b/src/GpuMetrics.cpp index 965f8a72b..241131bc1 100644 --- a/src/GpuMetrics.cpp +++ b/src/GpuMetrics.cpp @@ -38,6 +38,7 @@ Scope::~Scope() { } Reason::Reason(const char* name) : previous(reason) { reason = name; } Reason::~Reason() { reason = previous; } +const char* CurrentReason() { return reason; } void Record(Event event) { if (!current) return; current->counters->values[event].fetch_add(1, std::memory_order_relaxed); diff --git a/src/GpuMetrics.h b/src/GpuMetrics.h index fddc941fb..97f260aa5 100644 --- a/src/GpuMetrics.h +++ b/src/GpuMetrics.h @@ -34,6 +34,8 @@ class Reason { }; void Record(Event event); void RecordRender(const Frame& frame); +/// The current thread's reason for CPU work ("Other" outside a Reason scope). +const char* CurrentReason(); } #endif } diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 05fe84499..71bd0dfc6 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include extern "C" { #include @@ -49,9 +50,14 @@ template T glFunction(QOpenGLContext* context, const char* name) { return function; } #ifdef _WIN32 -constexpr auto kMemoryHandle = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT; -constexpr auto kSemaphoreHandle = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; -constexpr GLenum kGLHandle = GL_HANDLE_TYPE_OPAQUE_WIN32_EXT; +// GpuDebug: OPENSHOT_PRESENTATION_HANDLE=kmt shares through the older global +// (KMT) handles, which are never closed, instead of NT handles. +const bool kKmtHandles = GpuDebugSetting("OPENSHOT_PRESENTATION_HANDLE") == "kmt"; +const auto kMemoryHandle = kKmtHandles ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT + : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT; +const auto kSemaphoreHandle = kKmtHandles ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT + : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT; +const GLenum kGLHandle = kKmtHandles ? GL_HANDLE_TYPE_OPAQUE_WIN32_KMT_EXT : GL_HANDLE_TYPE_OPAQUE_WIN32_EXT; constexpr const char* kMemoryExtension = "GL_EXT_memory_object_win32"; constexpr const char* kSemaphoreExtension = "GL_EXT_semaphore_win32"; using ImportMemory = PFNGLIMPORTMEMORYWIN32HANDLEEXTPROC; @@ -103,6 +109,11 @@ struct SharedTarget { VkBuffer probe = VK_NULL_HANDLE; VkDeviceMemory probe_memory = VK_NULL_HANDLE; const unsigned char* probe_pixel = nullptr; +#ifdef _WIN32 + // Imported Win32 handles stay open while OpenGL uses them: a driver may + // resolve the handle after the import call returns. + std::vector handles; +#endif }; struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { QPointer widget; @@ -189,6 +200,9 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { if (target.probe) vkDestroyBuffer(target.device, target.probe, nullptr); if (target.probe_memory) vkFreeMemory(target.device, target.probe_memory, nullptr); } +#ifdef _WIN32 + for (auto handle : target.handles) CloseHandle(handle); +#endif av_buffer_unref(&target.device_ref); target = SharedTarget(); } @@ -295,7 +309,11 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { std::to_string(requirements.memoryRequirements.size) + "; memory type " + std::to_string(memoryType) + " flags=" + flags + "; dedicated prefers=" + std::to_string(dedicated.prefersDedicatedAllocation) + " requires=" + std::to_string(dedicated.requiresDedicatedAllocation) + " used=" + - std::to_string(useDedicated) + "; override=" + forceDedicated.toStdString(), Logger::LevelInfo); + std::to_string(useDedicated) + "; override=" + forceDedicated.toStdString() +#ifdef _WIN32 + + "; handles=" + (kKmtHandles ? "kmt" : "nt") +#endif + , Logger::LevelInfo); createProbe(memoryProperties); } // GpuDebug: check OpenGL errors after each import step, not only at the end. @@ -319,8 +337,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { HANDLE memoryHandle = nullptr; check(getMemoryHandle(target.device, &memoryInfo, &memoryHandle), "Export presentation memory"); importMemory(target.gl_memory, requirements.memoryRequirements.size, kGLHandle, memoryHandle); - // Win32 imports retain the resource, not ownership of the handle. - CloseHandle(memoryHandle); + if (!kKmtHandles) target.handles.push_back(memoryHandle); glStep("import memory"); #else VkMemoryGetFdInfoKHR memoryInfo{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; @@ -362,7 +379,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { HANDLE semaphoreHandle = nullptr; check(getSemaphoreHandle(target.device, &handleInfo, &semaphoreHandle), "Export presentation semaphore"); importSemaphore(target.gl_semaphores[direction], kGLHandle, semaphoreHandle); - CloseHandle(semaphoreHandle); + if (!kKmtHandles) target.handles.push_back(semaphoreHandle); #else VkSemaphoreGetFdInfoKHR handleInfo{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; handleInfo.semaphore = target.handoff[direction]; diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 30da0c2f0..034077cbd 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -4,6 +4,7 @@ #include "VulkanFrameImage.h" #include "GpuDevices.h" #include "Logger.h" +#include "GpuDebug.h" #include #include #include "VulkanCompositor.h" @@ -131,8 +132,14 @@ QImage VulkanFrameImage::Materialize() const { if (result < 0) LogReadbackFailure(impl->frame.get(), result); Check(result, "Read back Vulkan frame"); } - readback_count.fetch_add(1,std::memory_order_relaxed); + const auto readbacks = readback_count.fetch_add(1,std::memory_order_relaxed) + 1; gpu_metrics::Record(gpu_metrics::Readback); + // GpuDebug: say why frames are copied back ("Other" is work outside the + // timeline, such as thumbnails), for the first 20 and every 100th. + if (GpuDebug() && (readbacks <= 20 || readbacks % 100 == 0)) + Logger::Instance()->Log("GPU debug readback " + std::to_string(readbacks) + ": reason " + + gpu_metrics::CurrentReason() + "; " + std::to_string(GetWidth()) + "x" + std::to_string(GetHeight()), + Logger::LevelInfo); Check(av_frame_copy_props(cpu.get(), impl->frame.get()), "Copy Vulkan frame metadata"); QImage result(GetWidth(), GetHeight(), QImage::Format_RGBA8888); if (result.isNull()) throw std::bad_alloc(); From a577b589920b37648e48124ba3b9683e65d1f543 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 3 Nov 2026 08:51:38 -0600 Subject: [PATCH 33/50] Free CUDA-mapped NV12 export frames when the writer closes The NVENC converter's ring slots kept their last frames, which were mapped into CUDA, until process exit. By then CUDA had unloaded, so FFmpeg logged dozens of failed CUDA cleanup calls after the session ended. --- src/VulkanCompositor.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 5ec776f6c..fcc9d3d0d 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -428,24 +428,28 @@ std::shared_ptr RenderVulkanImage(std::shared_ptrsecond.pipeline; return pipeline->Render(*src,back.get(),matrix,opacity,clear,high_quality); } +namespace { +// Its ring slots keep recent NV12 frames, which NVENC export maps into CUDA. +thread_local std::unique_ptr nv12_pipeline; +thread_local std::tuple nv12_key{}; +} std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, const AVFrame* target) { auto src=std::dynamic_pointer_cast(source); if(!src) throw std::invalid_argument("NV12 conversion requires a Vulkan surface"); const auto* frame=src->GetFrame(); if(frame->width%2 || frame->height%2) throw std::invalid_argument("NV12 export requires even dimensions"); auto* fc=reinterpret_cast(frame->hw_frames_ctx->data); - using Key=std::tuple; - thread_local std::unique_ptr pipeline; - thread_local Key key{}; - Key requested{fc->device_ctx,frame->width,frame->height}; - if(!pipeline || key!=requested){ - pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true); - key=requested; + const std::tuple requested{fc->device_ctx,frame->width,frame->height}; + if(!nv12_pipeline || nv12_key!=requested){ + nv12_pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true); + nv12_key=requested; } - return pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black,false,target); + return nv12_pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black,false,target); } void ReleaseVulkanMemory() { + // Free CUDA-mapped NV12 frames while CUDA is loaded, not at process exit. + nv12_pipeline.reset(); ReleaseVulkanUploads(); #ifdef __APPLE__ ReleaseMacVulkanPools(); From a5967d3a800ee2a343504e6398ec58420a7b7136 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 3 Nov 2026 19:43:25 -0600 Subject: [PATCH 34/50] Count encoder readbacks in export GPU reporting --- src/FFmpegWriter.cpp | 1 + src/GpuMetrics.cpp | 11 +++++++++++ src/GpuMetrics.h | 4 ++++ tests/GpuMetrics.cpp | 23 +++++++++++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index c4018f5b8..a6b4c8f16 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -1140,6 +1140,7 @@ void FFmpegWriter::WriteHeader() { // Add a frame to the queue waiting to be encoded. void FFmpegWriter::WriteMeasuredFrame(ReaderBase* reader, int64_t number) { gpu_metrics::Scope scope(true); + gpu_metrics::RecordExportFrame(); WriteFrame(reader->GetFrame(number)); } diff --git a/src/GpuMetrics.cpp b/src/GpuMetrics.cpp index 241131bc1..d9a625a4d 100644 --- a/src/GpuMetrics.cpp +++ b/src/GpuMetrics.cpp @@ -11,6 +11,7 @@ namespace openshot { namespace gpu_metrics { struct Counters { std::array, Count> values{}; std::atomic resident{0}, requests{0}; + std::atomic export_frames{0}, export_resident{0}; std::array, 5> reasons{}; }; namespace { @@ -29,6 +30,10 @@ Scope::Scope(bool exporting) : previous(current), exceptions(std::uncaught_excep } Scope::~Scope() { if (counters) { + if (export_frame && std::uncaught_exceptions() == exceptions) { + counters->export_frames.fetch_add(1, std::memory_order_relaxed); + if (gpu && !readback) counters->export_resident.fetch_add(1, std::memory_order_relaxed); + } if (gpu && std::uncaught_exceptions() == exceptions) { counters->requests.fetch_add(1, std::memory_order_relaxed); if (!readback) counters->resident.fetch_add(1, std::memory_order_relaxed); @@ -39,6 +44,9 @@ Scope::~Scope() { Reason::Reason(const char* name) : previous(reason) { reason = name; } Reason::~Reason() { reason = previous; } const char* CurrentReason() { return reason; } +void RecordExportFrame() { + if (current) current->export_frame = true; +} void Record(Event event) { if (!current) return; current->counters->values[event].fetch_add(1, std::memory_order_relaxed); @@ -65,8 +73,11 @@ std::string GpuMetricsSnapshot(bool exporting) { for (int i = 0; i < Count; ++i) root[names[i]] = Json::UInt64(counters->values[i].load(std::memory_order_relaxed)); root["copy_free"] = Json::UInt64(counters->resident.load(std::memory_order_relaxed)); root["gpu_requests"] = Json::UInt64(counters->requests.load(std::memory_order_relaxed)); + root["export_frames"] = Json::UInt64(counters->export_frames.load(std::memory_order_relaxed)); + root["export_copy_free"] = Json::UInt64(counters->export_resident.load(std::memory_order_relaxed)); for (int i = 0; i < 5; ++i) root["readback_reasons"][reasons[i]] = Json::UInt64(counters->reasons[i].load(std::memory_order_relaxed)); root["definition"] = "Operation counts exclude cache hits; copy-free counts GPU work requests without GPU-to-CPU readbacks, not end-to-end host-free frames"; + root["export_copy_free_definition"] = "Measured writer frames with GPU work and no CPU readback; includes cached frames and excludes separate background rendering requests"; return root.toStyledString(); } void ResetGpuMetrics(bool exporting) { diff --git a/src/GpuMetrics.h b/src/GpuMetrics.h index 97f260aa5..e83977de2 100644 --- a/src/GpuMetrics.h +++ b/src/GpuMetrics.h @@ -18,8 +18,10 @@ class Scope { Scope* previous; std::shared_ptr counters; bool gpu = false, readback = false; + bool export_frame = false; int exceptions; friend void Record(Event event); + friend void RecordExportFrame(); public: explicit Scope(bool exporting); ~Scope(); @@ -34,6 +36,8 @@ class Reason { }; void Record(Event event); void RecordRender(const Frame& frame); +/// Mark a measured writer request, including one served from the timeline cache. +void RecordExportFrame(); /// The current thread's reason for CPU work ("Other" outside a Reason scope). const char* CurrentReason(); } diff --git a/tests/GpuMetrics.cpp b/tests/GpuMetrics.cpp index 237c4c793..93502c879 100644 --- a/tests/GpuMetrics.cpp +++ b/tests/GpuMetrics.cpp @@ -55,3 +55,26 @@ TEST_CASE("Failed GPU work is not a copy-free success", "[GpuMetrics]") { try { Scope scope(false); Record(DecodeGpu); throw 1; } catch (int) {} REQUIRE(snapshot()["gpu_requests"].asUInt64() == 0); } + +TEST_CASE("Export residency includes cached frames read back by a CPU encoder", "[GpuMetrics]") { + ResetGpuMetrics(true); + // The cache worker completes GPU work in a separate request. + { Scope render(true); Record(RenderGpu); } + { Scope writer(true); RecordExportFrame(); Reason reason("Encoder"); + Record(Readback); Record(EncodeCpu); } + auto value = snapshot(true); + REQUIRE(value["copy_free"].asUInt64() == 1); // Operation-only metric. + REQUIRE(value["export_frames"].asUInt64() == 1); + REQUIRE(value["export_copy_free"].asUInt64() == 0); + REQUIRE(value["readback_reasons"]["Encoder"].asUInt64() == 1); +} + +TEST_CASE("Export residency counts GPU writes and excludes failed requests and old sessions", "[GpuMetrics]") { + ResetGpuMetrics(true); + { Scope writer(true); RecordExportFrame(); Record(EncodeGpu); } + try { Scope writer(true); RecordExportFrame(); Record(EncodeGpu); throw 1; } catch (int) {} + REQUIRE(snapshot(true)["export_frames"].asUInt64() == 1); + REQUIRE(snapshot(true)["export_copy_free"].asUInt64() == 1); + { Scope old(true); RecordExportFrame(); Record(EncodeGpu); ResetGpuMetrics(true); } + REQUIRE(snapshot(true)["export_frames"].asUInt64() == 0); +} From e8916069621372735433aac5f663ab5d6872755d Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 3 Nov 2026 19:46:54 -0600 Subject: [PATCH 35/50] Prefer GPU-capable AV1 decoding when Vulkan supports it Select FFmpeg's native AV1 decoder on Vulkan devices advertising AV1 decode support, preserving the default decoder for software retries. Add hardware decode and output checks. Validation: three targeted AV1 tests pass on AMD Radeon 780M with Vulkan video enabled. Packaged FFmpeg builds including libdav1d still need validation. --- src/FFmpegReader.cpp | 9 +++++++++ src/GpuDevices.cpp | 11 +++++++++++ src/GpuDevices.h | 2 ++ tests/VulkanCompositor.cpp | 19 +++++++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 9a52d0842..9f0a5ace9 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -325,6 +325,15 @@ void FFmpegReader::Open() { // Get codec and codec context from stream const AVCodec *pCodec = avcodec_find_decoder(codecId); +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Full FFmpeg builds prefer libdav1d for AV1, but that decoder cannot + // produce Vulkan frames. Keep it as the default for software retries. + if (codecId == AV_CODEC_ID_AV1 && hw_de_on && VulkanAv1DecodeSupported()) { + const AVCodec* native_av1 = avcodec_find_decoder_by_name("av1"); + if (native_av1) + pCodec = native_av1; + } +#endif if (!pCodec) throw InvalidCodec("A valid video codec could not be found for this file.", path); AVDictionary *opts = NULL; diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 3c7c21d96..0af127a96 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -273,6 +273,17 @@ void ConfigureGpuEnvironment(const std::string& backend, const std::string& devi #endif } +bool VulkanAv1DecodeSupported() { + AVBufferRef* device = nullptr; + if (!VulkanRenderingEnabled() || CreateSharedVulkanDevice(&device) < 0) + return false; + const auto physical = PhysicalDevice(device); + const bool supported = Describe(physical).video_decode && + HasExtension(physical, "VK_KHR_video_decode_av1"); + av_buffer_unref(&device); + return supported; +} + std::vector ListGpuDevices() { const std::string active = ActiveGpuDevice().uuid; ListingInstance listing; diff --git a/src/GpuDevices.h b/src/GpuDevices.h index 9a9ee55cb..66de5e085 100644 --- a/src/GpuDevices.h +++ b/src/GpuDevices.h @@ -44,6 +44,8 @@ int CreateSharedVulkanDevice(AVBufferRef** output); std::string VulkanResultName(int result); /// Record a failed Vulkan call; VK_ERROR_DEVICE_LOST sets VulkanDeviceLost(). void ReportVulkanResult(int result); +/// Whether the active Vulkan driver advertises AV1 hardware decoding. +bool VulkanAv1DecodeSupported(); #endif } #endif diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 8aed37199..d9009d13c 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -77,6 +77,23 @@ TEST_CASE("Vulkan GPU listing reports the card OpenShot renders with", "[vulkan] for(const auto& name : encoders) CHECK(avcodec_find_encoder_by_name(name.c_str())); CHECK(openshot::GpuEncoders()==encoders); } +TEST_CASE("AV1 prefers Vulkan decoding over the default software decoder", "[vulkan][hardware][reader]") { + if (!VulkanAvailable()) return; + if (!openshot::VulkanAv1DecodeSupported()) { + WARN("Skipping Vulkan test: no Vulkan AV1 decode extension"); + return; + } + openshot::FFmpegReader reader(TEST_MEDIA_PATH "test_video_sync.mp4"); + reader.Open(); + auto frame = reader.GetFrame(90); + REQUIRE(frame->GetDeferredImage()); + CHECK(reader.HardwareDecodeSuccessful()); + auto image = frame->GetImage(); + REQUIRE(image); + CHECK(image->pixelColor(930, 820).red() > 250); + reader.Close(); +} + TEST_CASE("VA-API AV1 export accepts a requested bitrate", "[vulkan][hardware][writer]") { if(!VulkanAvailable()) return; const auto encoders=openshot::GpuEncoders(); @@ -99,6 +116,8 @@ TEST_CASE("VA-API AV1 export accepts a requested bitrate", "[vulkan][hardware][w CHECK(decoder->id==AV_CODEC_ID_AV1); auto image=reader.GetFrame(1)->GetImage(); REQUIRE(image); + if (openshot::VulkanAv1DecodeSupported()) + CHECK(reader.HardwareDecodeSuccessful()); CHECK(image->pixelColor(128,128).red()>200); CHECK(image->pixelColor(128,128).blue()<30); reader.Close(); From 8dff7c49265133fd69c5cdfa7c9c3abfe8025fab Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 3 Nov 2026 23:07:42 -0600 Subject: [PATCH 36/50] Use the Qt 5.15 SDK for Linux Vulkan builds --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8eb022e5..7b61a8ae7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ init-builders: linux-builder: stage: build-libopenshot variables: + OPENSHOT_QT_ROOT: /opt/openshot/qt-5.15.19 OPENCV_ROOT: /opt/opencv-4.13.0 OpenCV_DIR: /opt/opencv-4.13.0/lib/cmake/opencv4 # Runner image provides FFmpeg 7 (with Vulkan) and newer Vulkan headers in /opt @@ -48,11 +49,12 @@ linux-builder: - export PATH="$OPENCV_ROOT/bin:$PATH" - if [ ! -f "$FFMPEG_ROOT/env.sh" ]; then echo "ERROR runner is missing $FFMPEG_ROOT (FFmpeg 7 with Vulkan)"; exit 1; fi - . "$FFMPEG_ROOT/env.sh" + - . "$OPENSHOT_QT_ROOT/env.sh" - export FFMPEGDIR="$FFMPEG_ROOT" - bash ci/linux/check-ffmpeg-hardware.sh "$FFMPEG_ROOT" "$CI_PROJECT_DIR/build/ffmpeg-hardware-check" - mkdir -p build; cd build; - pkg-config --exists libpipewire-0.3 libspa-0.2 gio-2.0 gio-unix-2.0 || echo "WARNING PipeWire Wayland capture dependencies are missing; building without Wayland capture support" - - cmake -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ + - cmake -U "Qt5*_DIR" -U "*FFmpeg*" -U "*PC_av*" -U "*PC_sw*" -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -D"CMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/build/install-x64" -D"OpenCV_DIR:PATH=$OpenCV_DIR" -D"PYTHON_MODULE_PATH=python" -D"RUBY_MODULE_PATH=ruby" -DCMAKE_BUILD_TYPE:STRING=Release -DAPPIMAGE_BUILD=1 -DUSE_QT6=OFF -DUSE_SYSTEM_JSONCPP=0 -DENABLE_WAYLAND_CAPTURE=ON -DENABLE_GPU_FRAME=ON -D"Vulkan_INCLUDE_DIR:PATH=$VULKAN_HEADERS_ROOT/include" -D"CMAKE_CXX_FLAGS=-I$VULKAN_HEADERS_ROOT/include" ../ - cmake --build . --parallel $(nproc) # Some focal libraries link the system FFmpeg 4.2; never load two FFmpegs - ldd src/libopenshot.so | grep -E 'libav|libsw|vulkan' From e4c63eb618b51a488a805056646800932be7ffe5 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Wed, 4 Nov 2026 12:12:16 -0600 Subject: [PATCH 37/50] Bump Vulkan library builds to 1.2.0-dev1 and require audio 1.1.0 --- CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9addd9f22..c3db62bdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ For more information, please visit . set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/Modules") ################ PROJECT VERSION #################### -set(PROJECT_VERSION_FULL "1.1.1") +set(PROJECT_VERSION_FULL "1.2.0-dev1") set(PROJECT_SO_VERSION 33) # Remove the dash and anything following, to get the #.#.# version for project() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 648be5876..1b293b77e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -240,7 +240,7 @@ add_feature_info("Wayland screen capture" OPENSHOT_WAYLAND_CAPTURE "Use the opti # Find JUCE-based openshot Audio libraries if(NOT TARGET OpenShot::Audio) # Only load if necessary (not for integrated builds) - find_package(OpenShotAudio 1.0.2 REQUIRED) + find_package(OpenShotAudio 1.1.0 REQUIRED) endif() target_link_libraries(openshot PUBLIC OpenShot::Audio) From 0ee586bcd1fa9515981faf72d15c7363f6ce5b1c Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 5 Nov 2026 14:08:39 -0600 Subject: [PATCH 38/50] Accelerate common Vulkan effects and reuse GPU pipelines Keep common image effects and scope analysis on GPU surfaces, reuse pipelines across preview sizes, and retain per-submission color grading parameters. Restore Qt texture bindings after native presentation. Add coverage for animated effects, image sizing, pipeline reuse, GPU residency, and presentation state. Native presentation passes 219 assertions; broader compositor and CPU checks passed during development. A device-loss failure during GUI seek/edit/layout smoke testing remains unresolved. --- src/EffectBase.cpp | 15 +- src/EffectBase.h | 3 + src/FrameScope.cpp | 10 +- src/Qt/VulkanOpenGLPresenter.cpp | 19 +++ src/VulkanCompositor.cpp | 194 ++++++++++++++++++++---- src/VulkanCompositor.h | 25 ++++ src/effects/Blur.cpp | 36 +++-- src/effects/ColorGrade.cpp | 36 ++++- src/effects/Crop.cpp | 28 +++- src/effects/Mask.cpp | 25 ++++ src/effects/Saturation.cpp | 21 +++ src/vulkan/affine.comp | 77 ++++++++++ tests/QtImageReader.cpp | 27 ++++ tests/VulkanCompositor.cpp | 250 ++++++++++++++++++++++++++++++- tests/VulkanPresentation.cpp | 18 +++ 15 files changed, 732 insertions(+), 52 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 415e1c9af..d5975f415 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -502,6 +502,11 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ return empty_mask; } + return ResolveReaderMaskImage(target_image->width(),target_image->height(),frame_number); +} + +std::shared_ptr EffectBase::ResolveReaderMaskImage(int width,int height,int64_t frame_number) { + if (!mask_reader) return {}; @@ -515,8 +520,8 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ if (mask_reader->info.has_single_image && cached_single_mask_image && - cached_single_mask_width == target_image->width() && - cached_single_mask_height == target_image->height()) { + cached_single_mask_width == width && + cached_single_mask_height == height) { source_mask = cached_single_mask_image; used_cached_scaled = true; } @@ -541,12 +546,12 @@ std::shared_ptr EffectBase::GetMaskImage(std::shared_ptr target_ auto scaled_mask = std::make_shared( source_mask->scaled( - target_image->width(), target_image->height(), + width, height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); if (mask_reader->info.has_single_image) { cached_single_mask_image = scaled_mask; - cached_single_mask_width = target_image->width(); - cached_single_mask_height = target_image->height(); + cached_single_mask_width = width; + cached_single_mask_height = height; } return scaled_mask; } diff --git a/src/EffectBase.h b/src/EffectBase.h index 9da9c7149..41f89203d 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -93,6 +93,9 @@ namespace openshot return GetMaskImage(target_image, frame_number); } + /// Resolve reader artwork without requiring CPU pixels from the target frame. + std::shared_ptr ResolveReaderMaskImage(int width, int height, int64_t frame_number); + /// Optional override for effects that need custom mask behavior. virtual bool UseCustomMaskBlend(int64_t frame_number) const { return false; } diff --git a/src/FrameScope.cpp b/src/FrameScope.cpp index db2d54639..3aaa8e8b9 100644 --- a/src/FrameScope.cpp +++ b/src/FrameScope.cpp @@ -236,7 +236,15 @@ void FrameScope::analyze() { void FrameScope::analyze_video() { // Frame images are always QImage::Format_RGBA8888_Premultiplied (enforced // by Frame::AddImage). Pixel byte order is [R=0, G=1, B=2, A=3]. - std::shared_ptr image = frame->GetImage(); + std::shared_ptr image; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // Analysis is read-only. Do not replace the shared preview/cache surface + // with mutable CPU pixels just because the scopes need to inspect it. + if (auto surface=frame->GetDeferredImage()) + image=std::make_shared(surface->Materialize()); + else +#endif + image=frame->GetImage(); if (!image || image->isNull()) return; diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 71bd0dfc6..ee5ef88af 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -44,6 +44,24 @@ void check(VkResult result, const char* operation) { ReportVulkanResult(result); throw std::runtime_error(std::string(operation) + ": " + VulkanResultName(result)); } +// Qt shares this context with its widget texture cache. Leaving our video +// texture bound can interfere with later Qt popup/overlay texture uploads. +class TextureBindingGuard { + QOpenGLFunctions* gl; + GLint active = GL_TEXTURE0; + GLint texture = 0; +public: + explicit TextureBindingGuard(QOpenGLFunctions* functions) : gl(functions) { + gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE0); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &texture); + } + ~TextureBindingGuard() { + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glActiveTexture(active); + } +}; template T glFunction(QOpenGLContext* context, const char* name) { auto function = reinterpret_cast(context->getProcAddress(name)); if (!function) throw std::runtime_error(name); @@ -513,6 +531,7 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { return unavailable("OpenGL driver lacks external memory or semaphore sharing extensions"); auto gl = context->functions(); auto extra = context->extraFunctions(); + TextureBindingGuard textureBinding(gl); auto getUuid = glFunction(context, "glGetUnsignedBytei_vEXT"); GLint deviceCount = 0; gl->glGetIntegerv(GL_NUM_DEVICE_UUIDS_EXT, &deviceCount); diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index fcc9d3d0d..ef2a42a9c 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -9,6 +9,8 @@ #endif #include #include +#include +#include #include #include #include @@ -25,6 +27,7 @@ extern "C" { #include "encode_nv12.comp.h" namespace openshot { namespace { +std::atomic pipeline_creations{0}; void VkCheck(VkResult r, const char* operation) { if(r == VK_SUCCESS) return; ReportVulkanResult(r); @@ -46,7 +49,9 @@ constexpr int kSlots=4; // its last frame is released. struct OutputPools { std::mutex mutex; - std::map,AVBufferRef*> pools; + struct Entry { AVBufferRef* frames=nullptr; uint64_t used=0; }; + std::map,Entry> pools; + uint64_t clock=0; }; // Never destroyed: at process exit the Windows NVIDIA driver may already be // unloading, and freeing Vulkan frames from a static destructor crashes it. @@ -56,7 +61,9 @@ constexpr VkImageUsageFlags kOutputUsage=VK_IMAGE_USAGE_SAMPLED_BIT|VK_IMAGE_USA AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ auto& shared=SharedPools(); const std::lock_guard lock(shared.mutex); - auto& pool=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12}]; + auto& entry=shared.pools[{reinterpret_cast(device_ref->data),w,h,encode_nv12}]; + entry.used=++shared.clock; + auto& pool=entry.frames; if(!pool){ AVBufferRef* frames=av_hwframe_ctx_alloc(device_ref); if(!frames) throw std::bad_alloc(); @@ -73,14 +80,30 @@ AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ } pool=frames; } + // Resizing effects can produce a new size every frame. Keep retained pools + // bounded; frames still in use retain their own references to evicted pools. + while(shared.pools.size()>16) { + auto oldest=std::min_element(shared.pools.begin(),shared.pools.end(), + [](const auto& a,const auto& b){return a.second.usedsecond.frames); + shared.pools.erase(oldest); + } AVBufferRef* ref=av_buffer_ref(pool); if(!ref) throw std::bad_alloc(); return ref; } +struct EffectParameters { + int mode; // 1 saturation, 2 crop, 3 grading, 4 mask + std::array values{}; + const VulkanGradeParameters* grade=nullptr; +}; struct Parameters { float matrix_x[4], matrix_y[4], coeffs[4], extra[4], clear[4]; int sizes[4], modes[4]; + float effect[4]; }; +static_assert(sizeof(VulkanGradeParameters)==1045*sizeof(float), "Grading buffer must match shader scalar layout"); +static_assert(sizeof(Parameters)==128, "Vulkan guarantees 128 bytes of push constants"); struct LockedFrame { AVHWFramesContext* context; AVVulkanFramesContext* vkcontext; @@ -113,16 +136,44 @@ class Pipeline { VkCommandBuffer cmd=VK_NULL_HANDLE; VkFence fence=VK_NULL_HANDLE; bool busy=false; + VkBuffer grade_buffer=VK_NULL_HANDLE; + VkDeviceMemory grade_memory=VK_NULL_HANDLE; + void* grade_data=nullptr; + bool grade_coherent=false; std::vector views; FramePtr target; std::vector inputs; }; std::array ring; int next_slot=0; - int width,height; bool encode_nv12; + void CreateGradeBuffer(Slot& slot) { + VkBufferCreateInfo buffer{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + buffer.size=sizeof(VulkanGradeParameters); + buffer.usage=VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + VkCheck(vkCreateBuffer(device,&buffer,nullptr,&slot.grade_buffer),"Create grading parameter buffer"); + VkMemoryRequirements requirements; + vkGetBufferMemoryRequirements(device,slot.grade_buffer,&requirements); + VkPhysicalDeviceMemoryProperties memory; + vkGetPhysicalDeviceMemoryProperties(vkcontext->phys_dev,&memory); + uint32_t type=UINT32_MAX; + for(uint32_t i=0;i(device_ref->data); @@ -135,24 +186,25 @@ class Pipeline { VkCommandPoolCreateInfo ci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; ci.queueFamilyIndex=queue_family; ci.flags=VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; VkCheck(vkCreateCommandPool(device,&ci,nullptr,&commands),"Create command pool"); - std::array bindings{}; - for(int i=0;i<5;++i) { + std::array bindings{}; + for(int i=0;i<6;++i) { bindings[i].binding=i; bindings[i].descriptorCount=1; - bindings[i].descriptorType=(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + bindings[i].descriptorType=i==5?VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; bindings[i].stageFlags=VK_SHADER_STAGE_COMPUTE_BIT; } VkDescriptorSetLayoutCreateInfo di{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; - di.bindingCount=5; di.pBindings=bindings.data(); + di.bindingCount=6; di.pBindings=bindings.data(); VkCheck(vkCreateDescriptorSetLayout(device,&di,nullptr,&descriptors),"Create descriptors"); VkPushConstantRange range{VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(Parameters)}; VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; li.setLayoutCount=1; li.pSetLayouts=&descriptors; li.pushConstantRangeCount=1; li.pPushConstantRanges=⦥ VkCheck(vkCreatePipelineLayout(device,&li,nullptr,&layout),"Create pipeline layout"); - std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,4*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2*kSlots}}}; + std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,4*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,kSlots}}}; VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - pi.maxSets=kSlots; pi.poolSizeCount=2; pi.pPoolSizes=sizes.data(); + pi.maxSets=kSlots; pi.poolSizeCount=3; pi.pPoolSizes=sizes.data(); VkCheck(vkCreateDescriptorPool(device,&pi,nullptr,&pool),"Create descriptor pool"); for(auto& slot:ring){ + if(!encode_nv12) CreateGradeBuffer(slot); VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; ai.descriptorPool=pool; ai.descriptorSetCount=1; ai.pSetLayouts=&descriptors; VkCheck(vkAllocateDescriptorSets(device,&ai,&slot.set),"Allocate descriptors"); @@ -177,12 +229,16 @@ class Pipeline { const auto status=vkCreateComputePipelines(device,VK_NULL_HANDLE,1,&pci,nullptr,&pipeline); vkDestroyShaderModule(device,module,nullptr); VkCheck(status,"Create affine pipeline"); + pipeline_creations.fetch_add(1,std::memory_order_relaxed); }catch(...){Destroy();throw;} } ~Pipeline(){Destroy();} void Destroy(){ for(auto& slot:ring){ Retire(slot); + if(slot.grade_data) vkUnmapMemory(device,slot.grade_memory); + if(slot.grade_buffer) vkDestroyBuffer(device,slot.grade_buffer,nullptr); + if(slot.grade_memory) vkFreeMemory(device,slot.grade_memory,nullptr); if(slot.fence) vkDestroyFence(device,slot.fence,nullptr); } if(pipeline) vkDestroyPipeline(device,pipeline,nullptr); @@ -214,7 +270,8 @@ class Pipeline { return view; } std::shared_ptr Render(const VulkanFrameImage& input,const VulkanFrameImage* background, - const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false,const AVFrame* target=nullptr) { + int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false,const AVFrame* target=nullptr, + const EffectParameters* effect=nullptr) { const auto* src=input.GetFrame(); const auto* src_fc=reinterpret_cast(src->hw_frames_ctx->data); if(src_fc->device_ctx!=device_context) throw std::runtime_error("Vulkan source device mismatch"); @@ -272,6 +329,16 @@ class Pipeline { p.clear[3]=clear.alphaF();p.clear[0]=clear.redF()*p.clear[3];p.clear[1]=clear.greenF()*p.clear[3];p.clear[2]=clear.blueF()*p.clear[3]; p.modes[1]=back?1:0; p.modes[2]=high_quality?1:0; + if(effect) { + p.modes[3]=effect->mode; + std::copy_n(effect->values.begin(),4,p.effect); + if(effect->mode==3 || effect->mode==5) { + // Grading uses no background. Reuse clear and the unused matrix + // component to stay within Vulkan's guaranteed 128-byte limit. + std::copy_n(effect->values.begin()+4,4,p.clear); + p.matrix_x[3]=effect->values[8]; + } + } int plane_count=0; if(src_fc->sw_format==AV_PIX_FMT_RGBA){p.modes[0]=input.IsPremultiplied()?2:3;plane_count=1;} else if(src_fc->sw_format==AV_PIX_FMT_NV12){p.modes[0]=1;plane_count=1;} @@ -291,6 +358,15 @@ class Pipeline { Slot& slot=ring[next_slot]; next_slot=(next_slot+1)%kSlots; Retire(slot); + if(effect && effect->grade) { + // Each in-flight submission owns its parameters until its fence retires. + std::memcpy(slot.grade_data,effect->grade,sizeof(VulkanGradeParameters)); + if(!slot.grade_coherent) { + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; + range.memory=slot.grade_memory;range.size=VK_WHOLE_SIZE; + VkCheck(vkFlushMappedMemoryRanges(device,1,&range),"Flush grading parameters"); + } + } // Lock in stable address order, avoiding deadlocks when frames are shared. std::vector all{src,output.get()}; if(back && back->data[0]!=src->data[0]) all.push_back(back); @@ -335,13 +411,19 @@ class Pipeline { views[4]=make_view(of->img[separate?1:0],VK_FORMAT_R8G8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); }else views[4]=make_view(of->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); std::array images{}; - std::array writes{}; + std::array writes{}; for(int i=0;i<5;++i){ images[i].sampler=(i==4 || (encode_nv12 && i==3))?VK_NULL_HANDLE:sampler;images[i].imageView=views[i];images[i].imageLayout=VK_IMAGE_LAYOUT_GENERAL; writes[i].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;writes[i].dstSet=set;writes[i].dstBinding=i;writes[i].descriptorCount=1; writes[i].descriptorType=(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;writes[i].pImageInfo=&images[i]; } - vkUpdateDescriptorSets(device,5,writes.data(),0,nullptr); + VkDescriptorBufferInfo grade_buffer{slot.grade_buffer,0,sizeof(VulkanGradeParameters)}; + if(!encode_nv12) { + writes[5].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[5].dstSet=set;writes[5].dstBinding=5;writes[5].descriptorCount=1; + writes[5].descriptorType=VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;writes[5].pBufferInfo=&grade_buffer; + } + vkUpdateDescriptorSets(device,encode_nv12?5:6,writes.data(),0,nullptr); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};bi.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; VkCheck(vkBeginCommandBuffer(cmd,&bi),"Begin affine commands"); std::vector semaphores; @@ -402,36 +484,94 @@ class Pipeline { } }; } -std::shared_ptr RenderVulkanImage(std::shared_ptr source, - std::shared_ptr background,int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality){ +static std::shared_ptr RenderImage(std::shared_ptr source, + std::shared_ptr background,int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality, + const EffectParameters* effect){ auto src=std::dynamic_pointer_cast(source); auto back=std::dynamic_pointer_cast(background); if(!src || (background && !back)) throw std::invalid_argument("Vulkan compositor requires Vulkan surfaces"); if(width<=0 || height<=0) throw std::invalid_argument("Invalid compositor dimensions"); auto* fc=reinterpret_cast(src->GetFrame()->hw_frames_ctx->data); - using Key=std::tuple; - // Reader scaling and timeline transforms alternate output dimensions. Keep a - // bounded LRU so those operations do not recompile a pipeline on every frame. + using Key=AVHWDeviceContext*; + // Output dimensions are push constants, not pipeline state. Animated crop + // resizing must reuse the pipeline rather than compile one per frame size. struct CachedPipeline { std::unique_ptr pipeline; uint64_t used; }; thread_local std::map cache; thread_local uint64_t clock=0; - const Key requested{fc->device_ctx,width,height}; + const Key requested=fc->device_ctx; auto found=cache.find(requested); if(found==cache.end()){ if(cache.size()>=8){ auto oldest=std::min_element(cache.begin(),cache.end(),[](const auto& a,const auto& b){return a.second.used(fc->device_ref,width,height),0}).first; + found=cache.emplace(requested,CachedPipeline{std::make_unique(fc->device_ref),0}).first; } found->second.used=++clock; auto& pipeline=found->second.pipeline; - return pipeline->Render(*src,back.get(),matrix,opacity,clear,high_quality); + return pipeline->Render(*src,back.get(),width,height,matrix,opacity,clear,high_quality,nullptr,effect); +} +std::shared_ptr RenderVulkanImage(std::shared_ptr source, + std::shared_ptr background,int width,int height,const QTransform& matrix, + double opacity,const QColor& clear,bool high_quality) { + return RenderImage(source,background,width,height,matrix,opacity,clear,high_quality,nullptr); +} +std::shared_ptr SaturateVulkanImage(std::shared_ptr source, + const std::array& saturation) { + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{1}; + std::copy(saturation.begin(),saturation.end(),effect.values.begin()); + return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr CropVulkanImage(std::shared_ptr source, + const QRectF& destination,const QRectF& copy,const QRect& canvas) { + QTransform matrix; + matrix.translate(destination.x()-copy.x()-canvas.x(),destination.y()-copy.y()-canvas.y()); + const std::array bounds{float(destination.left()-canvas.x()),float(destination.top()-canvas.y()), + float(destination.right()-canvas.x()),float(destination.bottom()-canvas.y())}; + EffectParameters effect{2}; + std::copy(bounds.begin(),bounds.end(),effect.values.begin()); + return RenderImage(source,nullptr,canvas.width(),canvas.height(),matrix,1.0,Qt::transparent,false,&effect); +} +std::shared_ptr GradeVulkanImage(std::shared_ptr source, + const VulkanGradeParameters& parameters) { + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{3,parameters.controls,¶meters}; + return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr MaskVulkanImage(std::shared_ptr source, + std::shared_ptr mask,float contrast,int brightness,bool invert,bool replace) { + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{4,{contrast,float(brightness),float(invert),float(replace)}}; + return RenderImage(source,mask,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr BlurVulkanImage(std::shared_ptr source, + const QRect& area,int horizontal,int vertical,int iterations) { + const int width=source->GetWidth(),height=source->GetHeight(); + // Blur samples premultiplied RGBA byte values, as the CPU implementation does. + auto rgba=std::dynamic_pointer_cast(source); + if(!rgba) throw std::invalid_argument("Blur requires a Vulkan surface"); + const auto* context=reinterpret_cast(rgba->GetFrame()->hw_frames_ctx->data); + if(context->sw_format!=AV_PIX_FMT_RGBA || !rgba->IsPremultiplied()) + source=RenderVulkanImage(source,nullptr,width,height,QTransform(),1.0); + auto result=source; + for(int i=0;i nv12_pipeline; -thread_local std::tuple nv12_key{}; +thread_local AVHWDeviceContext* nv12_key=nullptr; } std::shared_ptr ConvertVulkanToNV12(std::shared_ptr source, const AVFrame* target) { auto src=std::dynamic_pointer_cast(source); @@ -439,14 +579,16 @@ std::shared_ptr ConvertVulkanToNV12(std::shared_ptrGetFrame(); if(frame->width%2 || frame->height%2) throw std::invalid_argument("NV12 export requires even dimensions"); auto* fc=reinterpret_cast(frame->hw_frames_ctx->data); - const std::tuple requested{fc->device_ctx,frame->width,frame->height}; + const auto requested=fc->device_ctx; if(!nv12_pipeline || nv12_key!=requested){ - nv12_pipeline=std::make_unique(fc->device_ref,frame->width,frame->height,true); + nv12_pipeline=std::make_unique(fc->device_ref,true); nv12_key=requested; } - return nv12_pipeline->Render(*src,nullptr,QTransform(),1.0,Qt::black,false,target); + return nv12_pipeline->Render(*src,nullptr,frame->width,frame->height,QTransform(),1.0,Qt::black,false,target); } +uint64_t VulkanPipelineCreationCount() { return pipeline_creations.load(std::memory_order_relaxed); } + void ReleaseVulkanMemory() { // Free CUDA-mapped NV12 frames while CUDA is loaded, not at process exit. nv12_pipeline.reset(); @@ -456,7 +598,7 @@ void ReleaseVulkanMemory() { #endif auto& shared=SharedPools(); const std::lock_guard lock(shared.mutex); - for(auto& item:shared.pools) av_buffer_unref(&item.second); + for(auto& item:shared.pools) av_buffer_unref(&item.second.frames); shared.pools.clear(); } diff --git a/src/VulkanCompositor.h b/src/VulkanCompositor.h index b5bbab487..a9f276c5d 100644 --- a/src/VulkanCompositor.h +++ b/src/VulkanCompositor.h @@ -4,8 +4,11 @@ #define OPENSHOT_VULKAN_COMPOSITOR_H #include "FrameImage.h" #include +#include +#include #include #include +#include namespace openshot { /// Affine transform and source-over compositing into an independently owned GPU frame. /// Inputs must be Vulkan surfaces from the same device; null background uses clear. @@ -14,6 +17,28 @@ std::shared_ptr RenderVulkanImage( std::shared_ptr source, std::shared_ptr background, int width, int height, const QTransform& source_to_output, double opacity, const QColor& clear = Qt::transparent, bool high_quality = false); +/// Apply common and per-channel saturation, preserving premultiplied alpha. +std::shared_ptr SaturateVulkanImage( + std::shared_ptr source, const std::array& saturation); +/// Copy a source rectangle into a transparent output, with optional resized canvas. +std::shared_ptr CropVulkanImage(std::shared_ptr source, + const QRectF& destination, const QRectF& copy, const QRect& canvas); +/// Per-frame grading parameters; curves retain the CPU's 256-entry lookup semantics. +struct VulkanGradeParameters { + std::array controls; // temperature, tint, exposure gain, contrast, shadows, highlights, saturation, vibrance, mix + std::array wheels; // RGB bias for global, shadows, midtones, highlights + std::array,4> curves; // master, red, green, blue +}; +std::shared_ptr GradeVulkanImage(std::shared_ptr source, + const VulkanGradeParameters& parameters); +/// Apply a reader mask using the existing brightness/contrast and inversion semantics. +std::shared_ptr MaskVulkanImage(std::shared_ptr source, + std::shared_ptr mask, float contrast, int brightness, bool invert, bool replace); +/// Separable box blur with clamped region edges and CPU-compatible compositing. +std::shared_ptr BlurVulkanImage(std::shared_ptr source, + const QRect& area, int horizontal, int vertical, int iterations); +/// Diagnostic count of successful compute pipeline creations. +uint64_t VulkanPipelineCreationCount(); /// Return pooled GPU frame memory once its frames are released, i.e. after /// clearing caches. Later renders create new pools as needed. void ReleaseVulkanMemory(); diff --git a/src/effects/Blur.cpp b/src/effects/Blur.cpp index 43f3ce8b3..3fb21b619 100644 --- a/src/effects/Blur.cpp +++ b/src/effects/Blur.cpp @@ -19,6 +19,11 @@ #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -81,9 +86,6 @@ void Blur::init_effect_details() // modified openshot::Frame object std::shared_ptr Blur::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the frame's image - std::shared_ptr frame_image = frame->GetImage(); - // Get the current blur radius int horizontal_radius_value = horizontal_radius.GetValue(frame_number); int vertical_radius_value = vertical_radius.GetValue(frame_number); @@ -91,31 +93,47 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr int iteration_value = iterations.GetInt(frame_number); (void) sigma_value; - int w = frame_image->width(); - int h = frame_image->height(); + int w = frame->GetWidth(); + int h = frame->GetHeight(); if (w <= 0 || h <= 0 || iteration_value <= 0) return frame; // Define area we're working on in terms of a QRect with QMargins applied. - QRect area(QPoint(0, 0), frame_image->size()); + QRect area(0,0,w,h); area = area.marginsRemoved({ int(clamp_margin(left.GetValue(frame_number)) * w), int(clamp_margin(top.GetValue(frame_number)) * h), int(clamp_margin(right.GetValue(frame_number)) * w), int(clamp_margin(bottom.GetValue(frame_number)) * h) }); - area = area.intersected(QRect(QPoint(0, 0), frame_image->size())); + area = area.intersected(QRect(0,0,w,h)); if (area.isEmpty()) return frame; + const int horizontal_area_radius=std::min(std::max(0,horizontal_radius_value),area.width()-1); + const int vertical_area_radius=std::min(std::max(0,vertical_radius_value),area.height()-1); + if (!horizontal_area_radius && !vertical_area_radius) return frame; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && !MaskReader() && MaskSourceId().empty()) { + try { + auto surface=frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface=UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(BlurVulkanImage(surface,area,horizontal_area_radius,vertical_area_radius,iteration_value)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan blur CPU fallback: ")+error.what()); + } + } +#endif + auto frame_image=frame->GetImage(); // Grab two copies of the image pixel data QImage image_copy = frame_image->copy(area); std::shared_ptr blur_image = std::make_shared(image_copy); std::shared_ptr frame_image_2 = std::make_shared(image_copy); const int area_w = blur_image->width(); const int area_h = blur_image->height(); - const int horizontal_area_radius = std::min(std::max(0, horizontal_radius_value), std::max(0, area_w - 1)); - const int vertical_area_radius = std::min(std::max(0, vertical_radius_value), std::max(0, area_h - 1)); bool blurred = false; // Loop through each iteration diff --git a/src/effects/ColorGrade.cpp b/src/effects/ColorGrade.cpp index 51e1f5ec0..feea618d1 100644 --- a/src/effects/ColorGrade.cpp +++ b/src/effects/ColorGrade.cpp @@ -20,6 +20,12 @@ using namespace openshot; +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif + namespace { constexpr float kInv255 = 1.0f / 255.0f; @@ -220,10 +226,6 @@ void ColorGrade::sync_lut_effect() { } std::shared_ptr ColorGrade::GetFrame(std::shared_ptr frame, int64_t frame_number) { - std::shared_ptr frame_image = frame->GetImage(); - if (!frame_image) - return frame; - const float temperature_value = std::max(-1.0f, std::min(1.0f, static_cast(temperature.GetValue(frame_number)))); const float tint_value = std::max(-1.0f, std::min(1.0f, static_cast(tint.GetValue(frame_number)))); const float exposure_value = std::max(-4.0f, std::min(4.0f, static_cast(exposure.GetValue(frame_number)))); @@ -261,6 +263,32 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptrGetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + const VulkanGradeParameters parameters{ + {temperature_value,tint_value,exposure_gain,contrast_factor,shadows_value,highlights_value, + saturation_value,vibrance_value,mix_value}, + {global_wheel.red_delta,global_wheel.green_delta,global_wheel.blue_delta, + shadows_wheel.red_delta,shadows_wheel.green_delta,shadows_wheel.blue_delta, + midtones_wheel.red_delta,midtones_wheel.green_delta,midtones_wheel.blue_delta, + highlights_wheel.red_delta,highlights_wheel.green_delta,highlights_wheel.blue_delta}, + {curve_all_lut,curve_red_lut,curve_green_lut,curve_blue_lut}}; + frame->AddDeferredImage(GradeVulkanImage(surface,parameters)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan grading CPU fallback: ") + error.what()); + } + } +#endif + auto frame_image = frame->GetImage(); + if (!frame_image) return frame; + static const std::array inv_alpha = [] { std::array lut{}; lut[0] = 0.0f; diff --git a/src/effects/Crop.cpp b/src/effects/Crop.cpp index cb5e1f102..350e5cb16 100644 --- a/src/effects/Crop.cpp +++ b/src/effects/Crop.cpp @@ -19,6 +19,11 @@ #include #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -53,9 +58,6 @@ void Crop::init_effect_details() // modified openshot::Frame object std::shared_ptr Crop::GetFrame(std::shared_ptr frame, int64_t frame_number) { - // Get the frame's image - std::shared_ptr frame_image = frame->GetImage(); - // Get current keyframe values double left_value = left.GetValue(frame_number); double top_value = top.GetValue(frame_number); @@ -66,7 +68,7 @@ std::shared_ptr Crop::GetFrame(std::shared_ptr double x_shift = x.GetValue(frame_number); double y_shift = y.GetValue(frame_number); - QSize sz = frame_image->size(); + QSize sz(frame->GetWidth(), frame->GetHeight()); // Compute destination rectangle to paint into QRectF paint_r( @@ -96,6 +98,24 @@ std::shared_ptr Crop::GetFrame(std::shared_ptr copy_r.setBottom(sz.height()); } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && !MaskReader() && MaskSourceId().empty()) { + const QRect canvas = resize ? paint_r.toRect() : QRect(QPoint(0,0),sz); + if (canvas.width()>0 && canvas.height()>0 && paint_r.width()>0 && paint_r.height()>0) { + try { + auto surface = frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(CropVulkanImage(surface,paint_r,copy_r,canvas)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan crop CPU fallback: ") + error.what()); + } + } + } +#endif + auto frame_image = frame->GetImage(); QImage cropped(sz, QImage::Format_RGBA8888_Premultiplied); cropped.fill(Qt::transparent); diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 51f15685a..21891346c 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -17,6 +17,11 @@ #include "ReaderBase.h" #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -55,6 +60,26 @@ void Mask::init_effect_details() // This method is required for all derived classes of EffectBase, and returns a // modified openshot::Frame object std::shared_ptr Mask::GetFrame(std::shared_ptr frame, int64_t frame_number) { + // An absent reader/generated mask is a no-op, including on deferred frames. + if (!MaskReader() && MaskSourceId().empty()) return frame; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && MaskSourceId().empty()) { + try { + auto mask=ResolveReaderMaskImage(frame->GetWidth(),frame->GetHeight(),frame_number); + if (!mask || mask->isNull()) return frame; + auto surface=frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface=UploadVulkanImage(*frame->GetImage()); + if (surface) { + const float factor=20.0f/std::max(0.5f,20.0f-float(contrast.GetValue(frame_number))); + frame->AddDeferredImage(MaskVulkanImage(surface,UploadVulkanImage(*mask),factor, + int(255*brightness.GetValue(frame_number)),mask_invert,replace_image)); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan mask CPU fallback: ")+error.what()); + } + } +#endif // Get frame image first std::shared_ptr frame_image = frame->GetImage(); if (!frame_image || frame_image->isNull()) diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 39bd6914a..698de37a6 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -14,6 +14,11 @@ #include "Exceptions.h" #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -123,6 +128,22 @@ void Saturation::init_effect_details() // modified openshot::Frame object std::shared_ptr Saturation::GetFrame(std::shared_ptr frame, int64_t frame_number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && !MaskReader() && MaskSourceId().empty()) { + try { + auto surface = frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(SaturateVulkanImage(surface, { + float(saturation.GetValue(frame_number)), float(saturation_R.GetValue(frame_number)), + float(saturation_G.GetValue(frame_number)), float(saturation_B.GetValue(frame_number))})); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan saturation CPU fallback: ") + error.what()); + } + } +#endif // Get the frame's image std::shared_ptr frame_image = frame->GetImage(); diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp index 98e684cac..bf236d9a3 100644 --- a/src/vulkan/affine.comp +++ b/src/vulkan/affine.comp @@ -5,6 +5,7 @@ layout(binding=1) uniform sampler2D plane_u; layout(binding=2) uniform sampler2D plane_v; layout(binding=3) uniform sampler2D background_tex; layout(binding=4, rgba8) uniform writeonly image2D output_img; +layout(std430,binding=5) readonly buffer GradingParameters { float values[]; } grading; layout(push_constant) uniform Parameters { vec4 matrix_x; vec4 matrix_y; @@ -13,6 +14,7 @@ layout(push_constant) uniform Parameters { vec4 clear; ivec4 sizes; // output width/height, source width/height ivec4 modes; // 0 planar, 1 NV12, 2 premult RGBA, 3 straight RGBA; background enabled + vec4 effect; // saturation controls, or crop bounds in output pixels } p; // Mitchell/Netravali family B=0, C=0.6 matches libswscale's default // SWS_BICUBIC kernel. Expand the footprint on minification to suppress aliasing. @@ -23,6 +25,7 @@ float cubic(float value) { return 0.0; } vec4 sample_source(sampler2D image, vec2 uv) { + if(p.modes.w==2 || p.modes.w==4 || p.modes.w==5) return texelFetch(image,ivec2(floor(uv*vec2(textureSize(image,0)))),0); if(p.modes.z==0) return texture(image,uv); ivec2 size=textureSize(image,0); vec2 pos=uv*vec2(size)-0.5; @@ -44,6 +47,14 @@ vec4 sample_source(sampler2D image, vec2 uv) { } return result/total; } +vec3 wheel_bias(int wheel) { + int start=9+wheel*3; + return vec3(grading.values[start],grading.values[start+1],grading.values[start+2]); +} +float grade_curve(int curve,float value) { + int index=clamp(int(value*255.0+0.5),0,255); + return grading.values[21+curve*256+index]; +} void main() { ivec2 pixel=ivec2(gl_GlobalInvocationID.xy); if(any(greaterThanEqual(pixel,p.sizes.xy))) return; @@ -65,8 +76,74 @@ void main() { y+p.color_coeffs.z*chroma.x+p.color_coeffs.w*chroma.y, y+p.extra.x*chroma.x),0,1),1); } + if(p.modes.w==1 && front.a>0.0) { + // Match Saturation's integer rounding and fixed-point brightness. + int alpha=int(round(front.a*255.0)); + ivec3 rgb=ivec3(round(front.rgb*255.0)); + if(alpha<255) rgb=ivec3(vec3(rgb)*(255.0/float(alpha))); + int weighted=(77*rgb.r*rgb.r+150*rgb.g*rgb.g+29*rgb.b*rgb.b+128)>>8; + float brightness=sqrt(float(weighted)); + rgb=clamp(ivec3(vec3(brightness)+(vec3(rgb)-brightness)*p.effect.x),ivec3(0),ivec3(255)); + vec3 channel=vec3(rgb)*sqrt(vec3(0.299,0.587,0.114)); + ivec3 red=ivec3(vec3(channel.r)+(vec3(rgb.r,0,0)-channel.r)*p.effect.y); + ivec3 green=ivec3(vec3(channel.g)+(vec3(0,rgb.g,0)-channel.g)*p.effect.z); + ivec3 blue=ivec3(vec3(channel.b)+(vec3(0,0,rgb.b)-channel.b)*p.effect.w); + rgb=clamp(red+green+blue,ivec3(0),ivec3(255)); + if(alpha<255) rgb=ivec3(vec3(rgb)*(float(alpha)/255.0)); + front.rgb=vec3(rgb)/255.0; + } + if(p.modes.w==3 && front.a>0.0) { + vec3 original=front.rgb/front.a; + vec3 rgb=original; + rgb.r=clamp(rgb.r+p.effect.x*0.125,0.0,1.0); + rgb.b=clamp(rgb.b-p.effect.x*0.125,0.0,1.0); + rgb=clamp(rgb+vec3(0.05,-0.1,0.05)*p.effect.y,0.0,1.0); + rgb=clamp(rgb*p.effect.z,0.0,1.0); + rgb=clamp((rgb-0.5)*p.effect.w+0.5,0.0,1.0); + float luma=dot(rgb,vec3(0.299,0.587,0.114)); + rgb=clamp(rgb+0.35*(p.clear.x*(1.0-luma)*(1.0-luma)+p.clear.y*luma*luma),0.0,1.0); + luma=dot(rgb,vec3(0.299,0.587,0.114)); + float centered=abs(luma-0.5)*2.0; + rgb=clamp(rgb+wheel_bias(0)+wheel_bias(1)*(1.0-luma)*(1.0-luma) + +wheel_bias(2)*clamp(1.0-centered*centered,0.0,1.0)+wheel_bias(3)*luma*luma,0.0,1.0); + luma=dot(rgb,vec3(0.299,0.587,0.114)); + float colorfulness=max(rgb.r,max(rgb.g,rgb.b))-min(rgb.r,min(rgb.g,rgb.b)); + float factor=p.clear.z*max(0.0,1.0+p.clear.w*(1.0-colorfulness)); + rgb=clamp(luma+(rgb-luma)*factor,0.0,1.0); + rgb=vec3(grade_curve(0,rgb.r),grade_curve(0,rgb.g),grade_curve(0,rgb.b)); + rgb=vec3(grade_curve(1,rgb.r),grade_curve(2,rgb.g),grade_curve(3,rgb.b)); + rgb=clamp(mix(original,rgb,p.matrix_x.w),0.0,1.0); + front.rgb=floor(rgb*front.a*255.0+0.5)/255.0; + } front*=p.extra.w; } + if(p.modes.w==2 && (any(lessThan(center,p.effect.xy)) || any(greaterThanEqual(center,p.effect.zw)))) + front=vec4(0); vec4 back=p.modes.y==1 ? texture(background_tex,center/vec2(p.sizes.xy)) : p.clear; + if(p.modes.w==5) { + ivec2 first=ivec2(p.effect.zw),last=ivec2(p.clear.xy); + if(all(greaterThanEqual(pixel,first)) && all(lessThanEqual(pixel,last))) { + int radius=int(p.effect.x); + ivec2 direction=p.effect.y==0.0?ivec2(1,0):ivec2(0,1); + ivec4 total=ivec4(0); + for(int offset=-radius;offset<=radius;++offset) + total+=ivec4(round(texelFetch(plane_y,clamp(pixel+direction*offset,first,last),0)*255.0)); + front=floor(vec4(total)/float(2*radius+1)+0.5)/255.0; + } + } + if(p.modes.w==4) { + ivec4 mask=ivec4(round(back*255.0)); + int gray=(mask.r*11+mask.g*16+mask.b*5)>>5; + // Preserve CPU rounding before truncation: do not fuse multiply/add. + precise float adjusted_value=p.effect.x*(float(gray)+p.effect.y-128.0)+128.0; + int adjusted=clamp(int(adjusted_value),0,255); + int alpha=clamp(mask.a-adjusted,0,255); + if(p.effect.w!=0.0) front=vec4(float(alpha)/255.0); + else { + if(p.effect.z!=0.0) alpha=255-alpha; + front=floor(round(front*255.0)*float(alpha)/255.0)/255.0; + } + } + if(p.modes.w==3 || p.modes.w==4 || p.modes.w==5) back=vec4(0); imageStore(output_img,pixel,front+back*(1-front.a)); } diff --git a/tests/QtImageReader.cpp b/tests/QtImageReader.cpp index 0c8647342..d41192b63 100644 --- a/tests/QtImageReader.cpp +++ b/tests/QtImageReader.cpp @@ -13,6 +13,7 @@ #include "openshot_catch.h" #include +#include #include #include "QtImageReader.h" @@ -156,3 +157,29 @@ TEST_CASE( "Max_Decode_Size_SVG_QtImageReader", "[libopenshot][qtimagereader]" ) r.Close(); } + +TEST_CASE("Artwork resolution follows maximum animation scale and preview size", "[libopenshot][qtimagereader]") { + QTemporaryDir directory; + const auto path=directory.filePath("large.png"); + QImage image(1024,1024,QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::red); + REQUIRE(image.save(path)); + QtImageReader reader(path.toStdString()); + Clip clip(&reader); + clip.scale_x.AddPoint(1,1.0,LINEAR); + clip.scale_x.AddPoint(10,2.0,LINEAR); + clip.scale_y=clip.scale_x; + Timeline timeline(128,128,Fraction(24,1),48000,2,LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + clip.Reader()->Open(); + auto first=reader.GetFrame(1)->GetImage(); + CHECK(first->size()==QSize(256,256)); + CHECK(reader.GetFrame(8)->GetImage()->cacheKey()==first->cacheKey()); + clip.scale_x.AddPoint(10,4.0,LINEAR); + clip.scale_y=clip.scale_x; + CHECK(reader.GetFrame(1)->GetWidth()==512); + timeline.preview_width=timeline.preview_height=64; + CHECK(reader.GetFrame(1)->GetWidth()==256); + timeline.Close(); +} diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index d9009d13c..737a7fcec 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -9,11 +9,18 @@ #include "VulkanEncodeConverter.h" #include "GpuDevices.h" #include "GpuMetrics.h" +#include "FrameScope.h" #include "FFmpegReader.h" #include "FFmpegWriter.h" #include #include #include "Clip.h" +#include "effects/Saturation.h" +#include "effects/Crop.h" +#include "effects/ColorGrade.h" +#include "effects/Mask.h" +#include "effects/Blur.h" +#include "QtImageReader.h" #include "Timeline.h" #include "DummyReader.h" #include @@ -329,9 +336,11 @@ TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan] auto first=openshot::RenderVulkanImage(source,nullptr,32,32,transform,0.5,Qt::blue); // A second render must not overwrite the first cached frame's GPU storage. auto second=openshot::RenderVulkanImage(source,nullptr,32,32,QTransform(),1.0,Qt::green); - gpu.reset();cpu.reset();pool.reset();device.reset();source.reset(); + gpu.reset();pool.reset();device.reset();source.reset(); CHECK(first->GetBytes() >= 32 * 32 * 4); QImage image=first->Materialize(); + // FFmpeg may import the upload memory asynchronously; retain it until completion. + cpu.reset(); const auto blended=image.pixelColor(12,12); CHECK(blended.red()>=126);CHECK(blended.red()<=129); CHECK(blended.green()==0); @@ -616,9 +625,19 @@ TEST_CASE("PNG clips upload once and keep animated transforms on Vulkan", "[vulk bitmap.fill(QColor(0, 0, 255, 128)); REQUIRE(bitmap.save(path)); openshot::Clip clip(path.toStdString()); - clip.scale_x = openshot::Keyframe(0.6); + clip.scale_x.AddPoint(1,0.6,openshot::LINEAR); + clip.scale_x.AddPoint(8,0.8,openshot::LINEAR); clip.scale_y = openshot::Keyframe(0.6); - clip.rotation = openshot::Keyframe(10); + clip.rotation.AddPoint(1,10,openshot::LINEAR); + clip.rotation.AddPoint(8,20,openshot::LINEAR); + openshot::Saturation saturation; + saturation.saturation.AddPoint(1,1.0,openshot::LINEAR); + saturation.saturation.AddPoint(8,1.5,openshot::LINEAR); + openshot::Crop crop; + crop.left.AddPoint(1,0.0,openshot::LINEAR); + crop.left.AddPoint(8,0.125,openshot::LINEAR); + clip.AddEffect(&saturation); + clip.AddEffect(&crop); openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); timeline.AddClip(&clip); timeline.Open(); @@ -689,3 +708,228 @@ TEST_CASE("Saved GPU preferences reach the native runtime environment", "[vulkan CHECK(openshot::VulkanRenderingEnabled()); CHECK(std::getenv("OPENSHOT_VULKAN_DEVICE") == nullptr); } + +namespace { +void ProcessCpuEffect(openshot::EffectBase& effect, std::shared_ptr frame, int number) { + struct Restore { + std::string backend, device; + ~Restore() { openshot::ConfigureGpuEnvironment(backend,device); } + } restore{std::getenv("OPENSHOT_RENDER_BACKEND") ? std::getenv("OPENSHOT_RENDER_BACKEND") : "", + std::getenv("OPENSHOT_VULKAN_DEVICE") ? std::getenv("OPENSHOT_VULKAN_DEVICE") : ""}; + openshot::ConfigureGpuEnvironment("cpu",restore.device); + effect.ProcessFrame(frame,number); +} +void CheckVulkanEffect(openshot::EffectBase& effect, const QImage& pixels, + std::shared_ptr source, int number, int tolerance) { + auto cpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + auto gpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + cpu->AddImage(std::make_shared(pixels.copy())); + gpu->AddDeferredImage(source); + ProcessCpuEffect(effect,cpu,number); + const auto readbacks=openshot::VulkanFrameReadbackCount(); + effect.ProcessFrame(gpu,number); + REQUIRE(gpu->GetDeferredImage()); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + auto expected=cpu->GetImage(),actual=gpu->GetImage(); + REQUIRE(actual->size()==expected->size()); + int max_error=0; + for (int y=0;yheight();++y) + for (int x=0;xwidth()*4;++x) + max_error=std::max(max_error,std::abs(int(expected->constScanLine(y)[x])-int(actual->constScanLine(y)[x]))); + INFO("frame=" << number << " max byte error=" << max_error); + CHECK(max_error<=tolerance); +} + +} + +TEST_CASE("Vulkan saturation matches CPU for animated channels and alpha", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels(32, 16, QImage::Format_RGBA8888_Premultiplied); + for (int y=0; y> frames; + for(int width=16;width<48;++width) { + frames.push_back(openshot::RenderVulkanImage(source,nullptr,width,32,QTransform(),1.0)); + REQUIRE(frames.back()->GetWidth()==width); + } + CHECK(openshot::VulkanPipelineCreationCount()-created<=1); + // Old frames remain valid after both submission slots and retained pools turn over. + CHECK(frames.front()->Materialize().pixelColor(8,16)==QColor(Qt::red)); + CHECK(frames.back()->Materialize().pixelColor(8,16)==QColor(Qt::red)); +} + +TEST_CASE("Queued grading frames retain their own wheel parameters", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels(16,16,QImage::Format_RGBA8888_Premultiplied); + pixels.fill(QColor(100,80,60)); + auto source=openshot::UploadVulkanImage(pixels); + openshot::VulkanGradeParameters parameters{}; + parameters.controls={0,0,1,1,0,0,1,0,1}; + for(auto& curve:parameters.curves) + for(int i=0;i<256;++i) curve[i]=float(i)/255.0f; + std::vector> frames; + for(int i=0;i<12;++i) { + parameters.wheels[0]=float(i)/32.0f; + frames.push_back(openshot::GradeVulkanImage(source,parameters)); + } + for(int i=0;i<12;++i) { + const auto color=frames[i]->Materialize().pixelColor(8,8); + CHECK(std::abs(color.red()-int(100+255.0*i/32.0+0.5))<=1); + CHECK(color.green()==80); + CHECK(color.blue()==60); + } +} + +TEST_CASE("Scope analysis preserves the cached GPU preview surface", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels(16,16,QImage::Format_RGBA8888_Premultiplied); + pixels.fill(Qt::red); + auto source=openshot::UploadVulkanImage(pixels); + auto frame=std::make_shared(1,16,16,"transparent"); + frame->AddDeferredImage(source); + openshot::FrameScope scope; + scope.SetFrame(frame); + CHECK(scope.HasVideo()); + CHECK(frame->GetDeferredImage()==source); +} diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 1c0652022..5c876598c 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -9,6 +9,7 @@ #include "Qt/VulkanOpenGLPresenter.h" #include #include +#include #include #include #include @@ -21,6 +22,7 @@ class Preview : public QOpenGLWidget { public: openshot::VulkanOpenGLPresenter presenter{reinterpret_cast(this)}; bool nativeDraw = false; + bool textureStatePreserved = false; void initializeGL() override { auto* current = context(); typedef const GLubyte* (APIENTRYP GetString)(GLenum); @@ -37,7 +39,22 @@ class Preview : public QOpenGLWidget { QPainter painter(this); painter.fillRect(rect(), Qt::blue); painter.beginNativePainting(); + // Qt also uploads textures for popup labels and screenshot overlays. + // Native drawing must leave its texture binding intact. + auto* gl = context()->functions(); + GLuint qtTexture = 0; + gl->glGenTextures(1, &qtTexture); + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, qtTexture); + gl->glActiveTexture(GL_TEXTURE1); nativeDraw = presenter.Draw(0, 0, width() * devicePixelRatioF(), height() * devicePixelRatioF()); + GLint active = 0, binding = 0; + gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE0); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &binding); + textureStatePreserved = binding == GLint(qtTexture) && active == GL_TEXTURE1; + gl->glBindTexture(GL_TEXTURE_2D, 0); + gl->glDeleteTextures(1, &qtTexture); painter.endNativePainting(); painter.fillRect(QRect(0, 0, 20, 20), Qt::green); } @@ -144,6 +161,7 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ REQUIRE(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); } REQUIRE(widget.nativeDraw); + CHECK(widget.textureStatePreserved); CHECK(widget.presenter.NativeDraws() >= 64); CHECK(widget.presenter.FallbackFrames() == 0); CHECK(widget.presenter.FallbackReason().empty()); From 29dd35870d9cb8b2ee7d5bf7e7926e3716a1fcd7 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 5 Nov 2026 14:26:56 -0600 Subject: [PATCH 39/50] Avoid AMD Linux Vulkan host-import failures and protect CPU recovery Patch FFmpeg 7.1 dependency builds to use staging memory for AMD Linux CPU transfers and correct the staging download copy direction. Preserve GPU-to-GPU video paths. Detect device loss hidden by FFmpeg transfer errors and prevent preview fallback from reading dead GPU frames. Verify the original reproducer, GUI smoke sequence, 906 compositor assertions, and 222 presentation assertions. --- ci/linux/setup-ffmpeg-vulkan.sh | 4 ++- ci/patch-ffmpeg-vulkan.sh | 11 +++++++ ci/patches/ffmpeg-vulkan-readback.patch | 42 ++++++++++++++++++++++++ ci/windows/setup-ffmpeg-windows-final.sh | 2 ++ src/Qt/VulkanOpenGLPresenter.cpp | 12 +++++-- src/VulkanFrameImage.cpp | 3 ++ tests/VulkanPresentation.cpp | 6 ++++ 7 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 ci/patch-ffmpeg-vulkan.sh create mode 100644 ci/patches/ffmpeg-vulkan-readback.patch diff --git a/ci/linux/setup-ffmpeg-vulkan.sh b/ci/linux/setup-ffmpeg-vulkan.sh index 88012992b..80d4c5008 100755 --- a/ci/linux/setup-ffmpeg-vulkan.sh +++ b/ci/linux/setup-ffmpeg-vulkan.sh @@ -5,6 +5,7 @@ # Not enabled: libwavpack (removed from FFmpeg 5+, native decoder used), # chromaprint (focal's library links FFmpeg 4.2, loading two FFmpegs at once). set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) FFMPEG_VERSION=${FFMPEG_VERSION:-7.1.3} VULKAN_HEADERS_TAG=${VULKAN_HEADERS_TAG:-v1.4.347} @@ -21,7 +22,7 @@ export DEBIAN_FRONTEND=noninteractive if [ "$SKIP_APT" != 1 ]; then sudo apt-get update sudo apt-get install -y --no-install-recommends \ - build-essential pkg-config nasm yasm git curl xz-utils cmake \ + build-essential pkg-config nasm yasm git curl xz-utils cmake patch \ glslang-tools libvulkan-dev \ libx264-dev libx265-dev libvpx-dev meson ninja-build \ libmp3lame-dev libopus-dev libvorbis-dev libtheora-dev libspeex-dev \ @@ -177,6 +178,7 @@ if [ ! -d "ffmpeg-$FFMPEG_VERSION" ]; then mv "FFmpeg-n$FFMPEG_VERSION" "ffmpeg-$FFMPEG_VERSION" fi cd "ffmpeg-$FFMPEG_VERSION" +bash "$script_dir/../patch-ffmpeg-vulkan.sh" "$PWD" export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" ./configure --prefix="$PREFIX" --enable-gpl --enable-version3 \ --enable-shared --disable-static --disable-doc \ diff --git a/ci/patch-ffmpeg-vulkan.sh b/ci/patch-ffmpeg-vulkan.sh new file mode 100644 index 000000000..3155759ae --- /dev/null +++ b/ci/patch-ffmpeg-vulkan.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Apply the FFmpeg 7.1 Vulkan host-import and staging-copy fixes. +set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +source_dir=${1:?Specify the FFmpeg source directory} +fix="$script_dir/patches/ffmpeg-vulkan-readback.patch" +if patch --dry-run --reverse --silent -d "$source_dir" -p1 < "$fix" >/dev/null 2>&1; then + echo "FFmpeg Vulkan readback fixes already applied" +else + patch --forward -d "$source_dir" -p1 < "$fix" +fi diff --git a/ci/patches/ffmpeg-vulkan-readback.patch b/ci/patches/ffmpeg-vulkan-readback.patch new file mode 100644 index 000000000..2f4da0716 --- /dev/null +++ b/ci/patches/ffmpeg-vulkan-readback.patch @@ -0,0 +1,42 @@ +--- a/libavutil/hwcontext_vulkan.c ++++ b/libavutil/hwcontext_vulkan.c +@@ -3813,13 +3813,15 @@ + } + } + +- for (int i = 0; i < planes; i++) +- av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, +- region[i].bufferRowLength, +- swf->data[i], +- swf->linesize[i], +- swf->linesize[i], +- region[i].imageExtent.height); ++ for (int i = 0; i < planes; i++) { ++ if (upload) ++ av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, region[i].bufferRowLength, ++ swf->data[i], swf->linesize[i], swf->linesize[i], region[i].imageExtent.height); ++ else ++ av_image_copy_plane(swf->data[i], swf->linesize[i], ++ vkbuf->mapped_mem + region[i].bufferOffset, region[i].bufferRowLength, ++ swf->linesize[i], region[i].imageExtent.height); ++ } + + if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && upload) { + ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1, +@@ -4079,7 +4081,15 @@ + return AVERROR(EINVAL); + + /* Setup buffers first */ +- if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) { ++ /* AMD Linux host-pointer imports can fail submission with EFAULT and ++ * lose the device. Use owned staging memory for CPU transfers there. ++ * GPU-to-GPU decode/render/presentation/export paths are unchanged. */ ++ int use_host_import = !!(p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY); ++#ifdef __linux__ ++ if (p->props.properties.vendorID == 0x1002) ++ use_host_import = 0; ++#endif ++ if (use_host_import) { + err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload); + if (err >= 0) + host_mapped = 1; diff --git a/ci/windows/setup-ffmpeg-windows-final.sh b/ci/windows/setup-ffmpeg-windows-final.sh index 0e309f0c5..a65e7ec86 100755 --- a/ci/windows/setup-ffmpeg-windows-final.sh +++ b/ci/windows/setup-ffmpeg-windows-final.sh @@ -2,6 +2,7 @@ # Run using C:\msys64\usr\bin\bash.exe. No pacman changes. # Usage: bash setup-ffmpeg-windows.sh mingw64|mingw32 set -euo pipefail +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) arch=${1:?Specify mingw64 or mingw32} case "$arch" in mingw64) cpu=x86_64 ;; mingw32) cpu=x86 ;; *) exit 2 ;; esac version=7.1.3 @@ -23,6 +24,7 @@ if [ ! -f "FFmpeg-n$version.tar.gz" ]; then fi printf '%s %s\n' e0b04c4b43d7e6d67cb6710334fb513adf13ac860532f30e1d0ac4c231f232fb "FFmpeg-n$version.tar.gz" | sha256sum -c - if [ ! -d "$root/FFmpeg-n$version" ]; then tar xzf "FFmpeg-n$version.tar.gz"; fi +bash "$script_dir/../patch-ffmpeg-vulkan.sh" "$root/FFmpeg-n$version" cd "$work" opts=(--prefix="$prefix" --target-os=mingw32 --arch="$cpu" --enable-gpl --enable-version3 --enable-shared --disable-static --disable-doc diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index ee5ef88af..79a21364d 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -179,8 +179,16 @@ struct VulkanOpenGLPresenter::Impl : QObject, FramePresenter { fallback_delivered = true; ++fallback_frames; display_surface.reset(); - const QImage image = *frame->GetImage(); - QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + // A lost device cannot provide CPU pixels. Let the GUI clear its + // caches and request a fresh CPU-rendered frame instead. + if (VulkanDeviceLost() && frame->GetDeferredImage()) return; + try { + const QImage image = *frame->GetImage(); + QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + } catch (const std::exception& error) { + last_failure = std::string("GPU preview fallback could not read the frame: ") + error.what(); + Logger::Instance()->Log(last_failure, Logger::LevelWarning); + } } } void waitPending() { diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 034077cbd..4fe4d17eb 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -48,6 +48,9 @@ void LogReadbackFailure(const AVFrame* frame, int result) { auto fc = reinterpret_cast(frame->hw_frames_ctx->data); auto vfc = static_cast(fc->hwctx); auto vk = static_cast(fc->device_ctx->hwctx); + // FFmpeg translates failed submissions to AVERROR_EXTERNAL. Confirm the + // VkResult here so callers stop using the dead device immediately. + ReportVulkanResult(vkDeviceWaitIdle(vk->act_dev)); bool hostImport = false; for (int i = 0; i < vk->nb_enabled_dev_extensions; ++i) hostImport |= std::strcmp(vk->enabled_dev_extensions[i], VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) == 0; diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 5c876598c..4c69b6990 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -16,6 +16,7 @@ #include extern "C" { #include +#include } namespace { class Preview : public QOpenGLWidget { @@ -210,6 +211,11 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ CHECK(widget.presenter.FallbackFrames() == 0); CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::blue)); CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::red)); + // Recovery must not read pixels from a dead device or throw through paintGL. + openshot::ReportVulkanResult(VK_ERROR_DEVICE_LOST); + CHECK_NOTHROW(widget.presenter.Present(frame)); + CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); + CHECK(widget.presenter.FallbackReason().find("device lost") != std::string::npos); widget.presenter.Clear(); CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); CHECK_FALSE(widget.presenter.FallbackReason().empty()); From 47471e83bec2346ce3953e22076505d521f0be86 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Thu, 5 Nov 2026 19:39:55 -0600 Subject: [PATCH 40/50] Restore Mac preview texture state and stop drawing after device loss --- src/Qt/VulkanOpenGLPresenterMac.mm | 36 ++++++++++++++++++++++++++++-- tests/VulkanPresentation.cpp | 31 ++++++++++++++++++++----- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/Qt/VulkanOpenGLPresenterMac.mm b/src/Qt/VulkanOpenGLPresenterMac.mm index 0ca33ecb8..70c07b01a 100644 --- a/src/Qt/VulkanOpenGLPresenterMac.mm +++ b/src/Qt/VulkanOpenGLPresenterMac.mm @@ -5,6 +5,8 @@ #include "VideoRenderer.h" #include "../VulkanFrameImage.h" #include "../VulkanCompositor.h" +#include "../GpuDevices.h" +#include "../Logger.h" #include "../MacVulkanInterop.h" #include #include @@ -25,6 +27,25 @@ } namespace openshot { namespace { +// Qt shares the texture units with native painting. Restore both targets used +// by Qt and the IOSurface bridge, including when importing or drawing fails. +class TextureBindingGuard { + QOpenGLFunctions* gl; + GLint active = GL_TEXTURE0, texture = 0, rectangle = 0; +public: + explicit TextureBindingGuard(QOpenGLFunctions* functions) : gl(functions) { + gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE0); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &texture); + gl->glGetIntegerv(GL_TEXTURE_BINDING_RECTANGLE, &rectangle); + } + ~TextureBindingGuard() { + gl->glActiveTexture(GL_TEXTURE0); + gl->glBindTexture(GL_TEXTURE_2D, texture); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, rectangle); + gl->glActiveTexture(active); + } +}; bool SameDevice(CGLContextObj context, uint64_t registry) { if (!context || !registry) return false; GLint current = 0; @@ -60,8 +81,16 @@ explicit Impl(QWidget* value) : widget(value) { void fallback() { if (!fallback_delivered && frame && widget) { fallback_delivered = true; ++fallback_frames; display_surface.reset(); - const QImage image = *frame->GetImage(); - QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + // A dead GPU cannot provide CPU pixels. The next rendered frame + // will use the CPU path instead of reading this stale surface. + if (VulkanDeviceLost() && frame->GetDeferredImage()) return; + try { + const QImage image = *frame->GetImage(); + QMetaObject::invokeMethod(widget, "present", Qt::QueuedConnection, Q_ARG(QImage, image)); + } catch (const std::exception& error) { + last_failure = std::string("GPU preview fallback could not read the frame: ") + error.what(); + Logger::Instance()->Log(last_failure, Logger::LevelWarning); + } } } void PresentFrame(const std::shared_ptr& value) override { @@ -83,6 +112,8 @@ bool draw(int x, int y, int width, int height) { if (!context || !cgl) return unavailable("No current Qt/CGL OpenGL context"); if (!frame || !widget) return unavailable("No video frame or preview widget"); if (fallback_delivered) return false; + if (VulkanDeviceLost()) + return unavailable("GPU device lost (driver reset or GPU timeout); the CPU is used for the rest of this session"); auto surface = display_surface ? display_surface : std::dynamic_pointer_cast(frame->GetDeferredImage()); if (!surface) return unavailable("Frame cannot be presented as a shared Mac Vulkan surface"); auto* fc = reinterpret_cast(surface->GetFrame()->hw_frames_ctx->data); @@ -102,6 +133,7 @@ bool draw(int x, int y, int width, int height) { if (!io) return unavailable("Core Video pixel buffer has no IOSurface"); auto* gl = context->functions(); auto* extra = context->extraFunctions(); + TextureBindingGuard textureBinding(gl); const bool core = context->format().profile() == QSurfaceFormat::CoreProfile; QOpenGLShaderProgram program; const char* vertex = core ? diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 4c69b6990..4fdb9caf2 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -43,19 +43,36 @@ class Preview : public QOpenGLWidget { // Qt also uploads textures for popup labels and screenshot overlays. // Native drawing must leave its texture binding intact. auto* gl = context()->functions(); - GLuint qtTexture = 0; - gl->glGenTextures(1, &qtTexture); + GLuint qtTextures[2] = {}; + gl->glGenTextures(2, qtTextures); gl->glActiveTexture(GL_TEXTURE0); - gl->glBindTexture(GL_TEXTURE_2D, qtTexture); + gl->glBindTexture(GL_TEXTURE_2D, qtTextures[0]); +#ifdef __APPLE__ + GLuint qtRectangle = 0; + gl->glGenTextures(1, &qtRectangle); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, qtRectangle); +#endif gl->glActiveTexture(GL_TEXTURE1); + gl->glBindTexture(GL_TEXTURE_2D, qtTextures[1]); nativeDraw = presenter.Draw(0, 0, width() * devicePixelRatioF(), height() * devicePixelRatioF()); - GLint active = 0, binding = 0; + GLint active = 0, binding = 0, otherBinding = 0; gl->glGetIntegerv(GL_ACTIVE_TEXTURE, &active); + gl->glActiveTexture(GL_TEXTURE1); + gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &otherBinding); + gl->glBindTexture(GL_TEXTURE_2D, 0); gl->glActiveTexture(GL_TEXTURE0); gl->glGetIntegerv(GL_TEXTURE_BINDING_2D, &binding); - textureStatePreserved = binding == GLint(qtTexture) && active == GL_TEXTURE1; + textureStatePreserved = binding == GLint(qtTextures[0]) && active == GL_TEXTURE1 && + otherBinding == GLint(qtTextures[1]); +#ifdef __APPLE__ + GLint rectangle = 0; + gl->glGetIntegerv(GL_TEXTURE_BINDING_RECTANGLE, &rectangle); + textureStatePreserved = textureStatePreserved && rectangle == GLint(qtRectangle); + gl->glBindTexture(GL_TEXTURE_RECTANGLE, 0); + gl->glDeleteTextures(1, &qtRectangle); +#endif gl->glBindTexture(GL_TEXTURE_2D, 0); - gl->glDeleteTextures(1, &qtTexture); + gl->glDeleteTextures(2, qtTextures); painter.endNativePainting(); painter.fillRect(QRect(0, 0, 20, 20), Qt::green); } @@ -158,6 +175,7 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ INFO("frame=" << i << " top=" << image.pixelColor(image.width()/2, image.height()/4).name().toStdString() << " bottom=" << image.pixelColor(image.width()/2, 3*image.height()/4).name().toStdString()); REQUIRE(widget.nativeDraw); + CHECK(widget.textureStatePreserved); REQUIRE(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::red)); REQUIRE(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::blue)); } @@ -216,6 +234,7 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ CHECK_NOTHROW(widget.presenter.Present(frame)); CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); CHECK(widget.presenter.FallbackReason().find("device lost") != std::string::npos); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); widget.presenter.Clear(); CHECK_FALSE(widget.presenter.Draw(0, 0, 128, 128)); CHECK_FALSE(widget.presenter.FallbackReason().empty()); From 57eac8bf317b045bca4d453cc3caab96809f9d6b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 15:50:52 -0600 Subject: [PATCH 41/50] Fix CPU-only builds with older FFmpeg Guard the reader's device-loss check and the writer's Vulkan encoder check so builds without GPU frames compile against FFmpeg versions that predate AV_PIX_FMT_VULKAN. --- src/FFmpegReader.cpp | 2 +- src/FFmpegWriter.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 9f0a5ace9..135e751cb 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -1337,7 +1337,7 @@ std::shared_ptr FFmpegReader::GetFrame(int64_t requested_frame) { // Prevent async calls to the remainder of this code const std::lock_guard lock(getFrameMutex); -#if USE_HW_ACCEL +#if USE_HW_ACCEL && defined(OPENSHOT_ENABLE_GPU_FRAME) // A lost GPU device cannot decode; reopen this reader on the CPU. if (hw_de_on && !force_sw_decode && VulkanDeviceLost()) { hw_decode_failed = true; diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index d6f4cf142..8d236d37b 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -2112,10 +2112,15 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { } } +#ifdef OPENSHOT_ENABLE_GPU_FRAME + const bool vulkan_encoder = hw_en_av_pix_fmt == AV_PIX_FMT_VULKAN; +#else + const bool vulkan_encoder = false; // Older FFmpeg has no AV_PIX_FMT_VULKAN +#endif switch (video_codec_ctx->codec_id) { case AV_CODEC_ID_H264: video_codec_ctx->max_b_frames = 0; // At least this GPU doesn't support b-frames - if (hw_en_av_pix_fmt != AV_PIX_FMT_VULKAN) { + if (!vulkan_encoder) { video_codec_ctx->profile = AV_PROFILE_H264_CONSTRAINED_BASELINE; av_opt_set(video_codec_ctx->priv_data, "preset", "slow", 0); if (hw_en_av_pix_fmt == AV_PIX_FMT_CUDA) From 53b42d8edbad55a0e6b7ea7ba5b72de45751dff9 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 15:50:52 -0600 Subject: [PATCH 42/50] Keep LUTs, effect masks and scopes on Vulkan Give ColorMap a Vulkan path, which ColorGrade's LUT now reuses, so LUTs no longer send grading back to the CPU. Blend effect masks on the GPU for effects that render on Vulkan, and count FrameScope histograms, waveforms and the vectorscope on the GPU, reading back only the counters. Masks match the CPU byte for byte and scope counts match exactly; LUTs stay within one byte. Failures fall back to the CPU. Add GPU-versus-CPU coverage for LUTs, masks and scopes, plus a masked, LUT-graded clip with scopes that needs no readbacks. --- src/EffectBase.cpp | 83 +++++++++--- src/EffectBase.h | 3 + src/FrameScope.cpp | 70 +++++++++-- src/VulkanCompositor.cpp | 251 +++++++++++++++++++++++++++---------- src/VulkanCompositor.h | 14 +++ src/effects/Blur.cpp | 2 +- src/effects/Blur.h | 3 + src/effects/ColorGrade.cpp | 28 ++--- src/effects/ColorGrade.h | 5 + src/effects/ColorMap.cpp | 35 +++++- src/effects/ColorMap.h | 5 + src/effects/Crop.cpp | 2 +- src/effects/Crop.h | 3 + src/effects/Saturation.cpp | 2 +- src/effects/Saturation.h | 3 + src/vulkan/affine.comp | 135 +++++++++++++++++--- tests/VulkanCompositor.cpp | 197 +++++++++++++++++++++++++++++ 17 files changed, 706 insertions(+), 135 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index d5975f415..1f8be85e2 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -26,6 +26,10 @@ #include "FFmpegReader.h" #include "QtImageReader.h" #include "Logger.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME + #include "VulkanCompositor.h" + #include "VulkanFrameImage.h" +#endif #include #include #include @@ -662,29 +666,72 @@ std::shared_ptr EffectBase::ProcessFrame(std::shared_ptrGetImage(); - if (!pre_image || pre_image->isNull()) - return GetFrame(frame, frame_number); + const auto blend_cpu = [&](std::shared_ptr original_image, std::shared_ptr output_frame) { + if (!output_frame) + return output_frame; + auto effected_image = output_frame->GetImage(); + if (!effected_image || effected_image->isNull() || + effected_image->size() != original_image->size()) + return output_frame; - const auto original_image = std::make_shared(pre_image->copy()); - auto output_frame = GetFrame(frame, frame_number); - if (!output_frame) - return output_frame; - auto effected_image = output_frame->GetImage(); - if (!effected_image || effected_image->isNull() || - effected_image->size() != original_image->size()) - return output_frame; + auto mask_image = GetMaskImage(effected_image, frame_number); + if (!mask_image || mask_image->isNull()) + return output_frame; - auto mask_image = GetMaskImage(effected_image, frame_number); - if (!mask_image || mask_image->isNull()) + if (UseCustomMaskBlend(frame_number)) + ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number); + else + BlendWithMask(original_image, effected_image, mask_image); return output_frame; + }; + +#ifdef OPENSHOT_ENABLE_GPU_FRAME + // GPU frames keep their input surface, and the mask blend stays on the GPU. + std::shared_ptr original_surface; + if (VulkanRenderingEnabled() && RendersOnVulkan() && !UseCustomMaskBlend(frame_number)) { + original_surface = frame->GetDeferredImage(); + // Clip images are uploaded as the effect would; the effect reuses this upload. + if (!original_surface && ParentClip() && frame->has_image_data) { + try { + original_surface = UploadVulkanImage(*frame->GetImage()); + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan effect mask upload failed: ") + error.what()); + } + } + } + if (original_surface) { + auto output_frame = GetFrame(frame, frame_number); + if (!output_frame) + return output_frame; + const int width = original_surface->GetWidth(), height = original_surface->GetHeight(); + if (auto effected = output_frame->GetDeferredImage()) { + // A resized result has no matching mask area, as on the CPU. + if (effected->GetWidth() != width || effected->GetHeight() != height) + return output_frame; + try { + // Generated masks only need the target size, not its pixels. + auto mask_image = ResolveMaskSourceEffect() + ? GetMaskImage(std::make_shared(width, height, QImage::Format_RGBA8888_Premultiplied), frame_number) + : ResolveReaderMaskImage(width, height, frame_number); + if (!mask_image || mask_image->isNull()) + return output_frame; + output_frame->AddDeferredImage(BlendMaskVulkanImage( + original_surface, effected, UploadVulkanImage(*mask_image), mask_invert)); + return output_frame; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan effect mask CPU fallback: ") + error.what()); + } + } + return blend_cpu(std::make_shared(original_surface->Materialize()), output_frame); + } +#endif - if (UseCustomMaskBlend(frame_number)) - ApplyCustomMaskBlend(original_image, effected_image, mask_image, frame_number); - else - BlendWithMask(original_image, effected_image, mask_image); + auto pre_image = frame->GetImage(); + if (!pre_image || pre_image->isNull()) + return GetFrame(frame, frame_number); - return output_frame; + const auto original_image = std::make_shared(pre_image->copy()); + return blend_cpu(original_image, GetFrame(frame, frame_number)); } /// Parent clip object of this reader (which can be unparented and NULL) diff --git a/src/EffectBase.h b/src/EffectBase.h index 41f89203d..de8857ad9 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -106,6 +106,9 @@ namespace openshot /// Optional override for effects that apply mask processing inside GetFrame(). virtual bool HandlesMaskInternally() const { return false; } + /// Effects with a Vulkan path keep masked GPU frames on the GPU. + virtual bool RendersOnVulkan() const { return false; } + public: /// Parent effect (which properties will set this effect properties) EffectBase* parentEffect; diff --git a/src/FrameScope.cpp b/src/FrameScope.cpp index 3aaa8e8b9..5ec41d85b 100644 --- a/src/FrameScope.cpp +++ b/src/FrameScope.cpp @@ -11,6 +11,10 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "FrameScope.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "Logger.h" +#endif #include #include @@ -236,21 +240,29 @@ void FrameScope::analyze() { void FrameScope::analyze_video() { // Frame images are always QImage::Format_RGBA8888_Premultiplied (enforced // by Frame::AddImage). Pixel byte order is [R=0, G=1, B=2, A=3]. - std::shared_ptr image; + std::shared_ptr image; + int width = 0; + int height = 0; #ifdef OPENSHOT_ENABLE_GPU_FRAME - // Analysis is read-only. Do not replace the shared preview/cache surface - // with mutable CPU pixels just because the scopes need to inspect it. - if (auto surface=frame->GetDeferredImage()) - image=std::make_shared(surface->Materialize()); - else + // Analysis is read-only. Do not replace the shared preview/cache surface + // with mutable CPU pixels just because the scopes need to inspect it. + auto surface = frame->GetDeferredImage(); + if (surface) { + width = surface->GetWidth(); + height = surface->GetHeight(); + } else #endif - image=frame->GetImage(); - if (!image || image->isNull()) + { + image = frame->GetImage(); + if (!image || image->isNull()) + return; + width = image->width(); + height = image->height(); + } + if (width <= 0 || height <= 0) return; video_present = true; - const int width = image->width(); - const int height = image->height(); int start_x = 0; int end_x = width; int start_y = 0; @@ -273,6 +285,44 @@ void FrameScope::analyze_video() { clipped_green = 0; clipped_blue = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (surface) { + try { + // Count on the GPU; only the counters return to the CPU. + const auto counts = ScopeVulkanImage(surface, QRect(start_x, start_y, video_width, video_height), + waveform_columns, waveform_bins, vectorscope_size); + auto next = counts.begin(); + for (auto* values : {&histogram_luma, &histogram_red, &histogram_green, &histogram_blue, + &waveform_luma, &waveform_red, &waveform_green, &waveform_blue, &vectorscope}) { + std::copy_n(next, values->size(), values->begin()); + next += static_cast(values->size()); + } + // The summary follows from the histograms. Average luma uses bin + // centers, within half a bin of the CPU's exact sum. + double weighted = 0.0; + int64_t pixels = 0; + for (int bin = 0; bin < 256; ++bin) { + pixels += histogram_luma[bin]; + weighted += static_cast(bin) * histogram_luma[bin]; + } + avg_luma = pixels > 0 ? weighted / (255.0 * static_cast(pixels)) : 0.0; + for (int bin = 0; bin < 3; ++bin) + clipped_shadows += histogram_luma[bin]; + for (int bin = 253; bin < 256; ++bin) { + clipped_highlights += histogram_luma[bin]; + clipped_red += histogram_red[bin]; + clipped_green += histogram_green[bin]; + clipped_blue += histogram_blue[bin]; + } + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan scope CPU fallback: ") + error.what()); + } + image = std::make_shared(surface->Materialize()); + if (image->isNull()) + return; + } +#endif const int bytes_per_line = image->bytesPerLine(); const unsigned char* bits = image->constBits(); const auto& inv_alpha = inv_alpha_lut(); diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index ef2a42a9c..1fc405e94 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -93,9 +94,13 @@ AVBufferRef* OutputPool(AVBufferRef* device_ref,int w,int h,bool encode_nv12){ return ref; } struct EffectParameters { - int mode; // 1 saturation, 2 crop, 3 grading, 4 mask + int mode; // 1 saturation, 2 crop, 3 grading, 4 mask, 5 blur, 6 LUT, 7 mask blend, 8 scope std::array values{}; - const VulkanGradeParameters* grade=nullptr; + const void* data=nullptr; // copied into the slot's shader buffer (binding 5) + size_t data_bytes=0; + std::shared_ptr data_owner; // unchanged data: skip the copy while a slot holds it + const VulkanFrameImage* mask=nullptr; // third input (binding 6) + std::vector* counts=nullptr; // scope: zeroed, counted on the GPU, then read back }; struct Parameters { float matrix_x[4], matrix_y[4], coeffs[4], extra[4], clear[4]; @@ -131,15 +136,22 @@ class Pipeline { VkSampler sampler=VK_NULL_HANDLE; // Consumers order submissions through the frames' timeline semaphores. // Keep encoder-owned targets alive until their views and work are retired. + struct Buffer { + VkBuffer buffer=VK_NULL_HANDLE; + VkDeviceMemory memory=VK_NULL_HANDLE; + void* mapped=nullptr; + VkDeviceSize size=0; + bool coherent=true; + }; struct Slot { VkDescriptorSet set=VK_NULL_HANDLE; VkCommandBuffer cmd=VK_NULL_HANDLE; VkFence fence=VK_NULL_HANDLE; bool busy=false; - VkBuffer grade_buffer=VK_NULL_HANDLE; - VkDeviceMemory grade_memory=VK_NULL_HANDLE; - void* grade_data=nullptr; - bool grade_coherent=false; + Buffer data; // host-written shader parameters and tables + Buffer counts; // GPU-only scope counters + Buffer staging; // host-read copy of the counters + std::shared_ptr data_owner; std::vector views; FramePtr target; std::vector inputs; @@ -147,30 +159,50 @@ class Pipeline { std::array ring; int next_slot=0; bool encode_nv12; - void CreateGradeBuffer(Slot& slot) { - VkBufferCreateInfo buffer{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; - buffer.size=sizeof(VulkanGradeParameters); - buffer.usage=VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; - VkCheck(vkCreateBuffer(device,&buffer,nullptr,&slot.grade_buffer),"Create grading parameter buffer"); + void Release(Buffer& b) { + if(b.mapped) vkUnmapMemory(device,b.memory); + if(b.buffer) vkDestroyBuffer(device,b.buffer,nullptr); + if(b.memory) vkFreeMemory(device,b.memory,nullptr); + b=Buffer{}; + } + // Grow-only. Choose the memory type with the most preferred properties: + // device-local for data shaders read per pixel, cached for host readback. + void Reserve(Buffer& b,VkDeviceSize size,VkBufferUsageFlags usage,VkMemoryPropertyFlags required,VkMemoryPropertyFlags preferred) { + if(b.size>=size) return; + Release(b); + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + info.size=size; info.usage=usage; + VkCheck(vkCreateBuffer(device,&info,nullptr,&b.buffer),"Create shader buffer"); VkMemoryRequirements requirements; - vkGetBufferMemoryRequirements(device,slot.grade_buffer,&requirements); + vkGetBufferMemoryRequirements(device,b.buffer,&requirements); VkPhysicalDeviceMemoryProperties memory; vkGetPhysicalDeviceMemoryProperties(vkcontext->phys_dev,&memory); - uint32_t type=UINT32_MAX; + uint32_t type=UINT32_MAX; int best=-1; for(uint32_t i=0;i(flags&preferred).count()); + if(score>best) { best=score; type=i; } } - if(type==UINT32_MAX) throw std::runtime_error("No host-visible grading parameter memory"); - slot.grade_coherent=memory.memoryTypes[type].propertyFlags&VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if(type==UINT32_MAX) throw std::runtime_error("No suitable Vulkan shader buffer memory"); + b.coherent=memory.memoryTypes[type].propertyFlags&VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; - allocation.allocationSize=requirements.size;allocation.memoryTypeIndex=type; - VkCheck(vkAllocateMemory(device,&allocation,nullptr,&slot.grade_memory),"Allocate grading parameter memory"); - VkCheck(vkBindBufferMemory(device,slot.grade_buffer,slot.grade_memory,0),"Bind grading parameter memory"); - VkCheck(vkMapMemory(device,slot.grade_memory,0,VK_WHOLE_SIZE,0,&slot.grade_data),"Map grading parameters"); + allocation.allocationSize=requirements.size; allocation.memoryTypeIndex=type; + VkCheck(vkAllocateMemory(device,&allocation,nullptr,&b.memory),"Allocate shader buffer"); + VkCheck(vkBindBufferMemory(device,b.buffer,b.memory,0),"Bind shader buffer"); + if(required&VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) + VkCheck(vkMapMemory(device,b.memory,0,VK_WHOLE_SIZE,0,&b.mapped),"Map shader buffer"); + b.size=size; + } + void ReserveData(Slot& slot,VkDeviceSize size) { + Reserve(slot.data,size,VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + } + void Sync(const Buffer& b,bool to_device) { + if(b.coherent) return; + VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; + range.memory=b.memory; range.size=VK_WHOLE_SIZE; + VkCheck(to_device?vkFlushMappedMemoryRanges(device,1,&range):vkInvalidateMappedMemoryRanges(device,1,&range),"Synchronize shader buffer"); } public: Pipeline(AVBufferRef* device_owner,bool encode=false):encode_nv12(encode) { @@ -186,25 +218,25 @@ class Pipeline { VkCommandPoolCreateInfo ci{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; ci.queueFamilyIndex=queue_family; ci.flags=VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; VkCheck(vkCreateCommandPool(device,&ci,nullptr,&commands),"Create command pool"); - std::array bindings{}; - for(int i=0;i<6;++i) { + std::array bindings{}; + for(int i=0;i<7;++i) { bindings[i].binding=i; bindings[i].descriptorCount=1; bindings[i].descriptorType=i==5?VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; bindings[i].stageFlags=VK_SHADER_STAGE_COMPUTE_BIT; } VkDescriptorSetLayoutCreateInfo di{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; - di.bindingCount=6; di.pBindings=bindings.data(); + di.bindingCount=7; di.pBindings=bindings.data(); VkCheck(vkCreateDescriptorSetLayout(device,&di,nullptr,&descriptors),"Create descriptors"); VkPushConstantRange range{VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(Parameters)}; VkPipelineLayoutCreateInfo li{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; li.setLayoutCount=1; li.pSetLayouts=&descriptors; li.pushConstantRangeCount=1; li.pPushConstantRanges=⦥ VkCheck(vkCreatePipelineLayout(device,&li,nullptr,&layout),"Create pipeline layout"); - std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,4*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,kSlots}}}; + std::array sizes{{{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,5*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,2*kSlots},{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,kSlots}}}; VkDescriptorPoolCreateInfo pi{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; pi.maxSets=kSlots; pi.poolSizeCount=3; pi.pPoolSizes=sizes.data(); VkCheck(vkCreateDescriptorPool(device,&pi,nullptr,&pool),"Create descriptor pool"); for(auto& slot:ring){ - if(!encode_nv12) CreateGradeBuffer(slot); + if(!encode_nv12) ReserveData(slot,sizeof(VulkanGradeParameters)); VkDescriptorSetAllocateInfo ai{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; ai.descriptorPool=pool; ai.descriptorSetCount=1; ai.pSetLayouts=&descriptors; VkCheck(vkAllocateDescriptorSets(device,&ai,&slot.set),"Allocate descriptors"); @@ -236,9 +268,7 @@ class Pipeline { void Destroy(){ for(auto& slot:ring){ Retire(slot); - if(slot.grade_data) vkUnmapMemory(device,slot.grade_memory); - if(slot.grade_buffer) vkDestroyBuffer(device,slot.grade_buffer,nullptr); - if(slot.grade_memory) vkFreeMemory(device,slot.grade_memory,nullptr); + for(auto* b:{&slot.data,&slot.counts,&slot.staging}) Release(*b); if(slot.fence) vkDestroyFence(device,slot.fence,nullptr); } if(pipeline) vkDestroyPipeline(device,pipeline,nullptr); @@ -281,6 +311,13 @@ class Pipeline { if(bc->device_ctx!=device_context || bc->sw_format!=AV_PIX_FMT_RGBA || back->width!=width || back->height!=height) throw std::runtime_error("Vulkan background must be matching RGBA surface"); } + const AVFrame* mask=effect && effect->mask?effect->mask->GetFrame():nullptr; + if(mask){ + const auto* mc=reinterpret_cast(mask->hw_frames_ctx->data); + if(mc->device_ctx!=device_context || mc->sw_format!=AV_PIX_FMT_RGBA || mask->width!=width || mask->height!=height) + throw std::runtime_error("Vulkan mask must be matching RGBA surface"); + } + const bool scope=effect && effect->counts; FramePtr output(av_frame_alloc()); if(!output) throw std::bad_alloc(); if(target){ @@ -332,12 +369,10 @@ class Pipeline { if(effect) { p.modes[3]=effect->mode; std::copy_n(effect->values.begin(),4,p.effect); - if(effect->mode==3 || effect->mode==5) { - // Grading uses no background. Reuse clear and the unused matrix - // component to stay within Vulkan's guaranteed 128-byte limit. - std::copy_n(effect->values.begin()+4,4,p.clear); - p.matrix_x[3]=effect->values[8]; - } + // Effects render over transparent. Reuse clear and the unused matrix + // component to stay within Vulkan's guaranteed 128-byte limit. + std::copy_n(effect->values.begin()+4,4,p.clear); + p.matrix_x[3]=effect->values[8]; } int plane_count=0; if(src_fc->sw_format==AV_PIX_FMT_RGBA){p.modes[0]=input.IsPremultiplied()?2:3;plane_count=1;} @@ -358,18 +393,26 @@ class Pipeline { Slot& slot=ring[next_slot]; next_slot=(next_slot+1)%kSlots; Retire(slot); - if(effect && effect->grade) { - // Each in-flight submission owns its parameters until its fence retires. - std::memcpy(slot.grade_data,effect->grade,sizeof(VulkanGradeParameters)); - if(!slot.grade_coherent) { - VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; - range.memory=slot.grade_memory;range.size=VK_WHOLE_SIZE; - VkCheck(vkFlushMappedMemoryRanges(device,1,&range),"Flush grading parameters"); - } + // Each in-flight submission owns its parameters until its fence retires. + // Tables such as LUTs are copied once while a slot still holds them. + if(effect && effect->data && (!effect->data_owner || slot.data_owner!=effect->data_owner)) { + ReserveData(slot,effect->data_bytes); + std::memcpy(slot.data.mapped,effect->data,effect->data_bytes); + Sync(slot.data,true); + slot.data_owner=effect->data_owner; + } + const VkDeviceSize count_bytes=scope?effect->counts->size()*sizeof(uint32_t):0; + if(scope) { + Reserve(slot.counts,count_bytes,VK_BUFFER_USAGE_STORAGE_BUFFER_BIT|VK_BUFFER_USAGE_TRANSFER_SRC_BIT|VK_BUFFER_USAGE_TRANSFER_DST_BIT, + 0,VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Reserve(slot.staging,count_bytes,VK_BUFFER_USAGE_TRANSFER_DST_BIT,VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, + VK_MEMORY_PROPERTY_HOST_CACHED_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } // Lock in stable address order, avoiding deadlocks when frames are shared. std::vector all{src,output.get()}; - if(back && back->data[0]!=src->data[0]) all.push_back(back); + for(const AVFrame* extra:{back,mask}) + if(extra && std::none_of(all.begin(),all.end(),[&](const AVFrame* f){return f->data[0]==extra->data[0];})) + all.push_back(extra); std::sort(all.begin(),all.end(),[](const AVFrame* a,const AVFrame* b){return std::less()(a->data[0],b->data[0]);}); std::vector> locks; for(auto* f:all) locks.emplace_back(new LockedFrame(f)); @@ -386,13 +429,13 @@ class Pipeline { // disappear when its writer closes, before this cached pipeline. slot.target.reset(av_frame_clone(output.get())); if(!slot.target) throw std::bad_alloc(); - for(const AVFrame* input_frame : {src, back}) { + for(const AVFrame* input_frame : {src, back, mask}) { if (!input_frame) continue; FramePtr retained(av_frame_clone(input_frame)); if (!retained) throw std::bad_alloc(); slot.inputs.push_back(std::move(retained)); } - std::array views{}; + std::array views{}; auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ auto view=View(image,format,aspect);owned_views.push_back(view);return view; }; @@ -410,20 +453,25 @@ class Pipeline { views[3]=make_view(of->img[0],VK_FORMAT_R8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); views[4]=make_view(of->img[separate?1:0],VK_FORMAT_R8G8_UNORM,separate?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); }else views[4]=make_view(of->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); - std::array images{}; - std::array writes{}; - for(int i=0;i<5;++i){ - images[i].sampler=(i==4 || (encode_nv12 && i==3))?VK_NULL_HANDLE:sampler;images[i].imageView=views[i];images[i].imageLayout=VK_IMAGE_LAYOUT_GENERAL; - writes[i].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;writes[i].dstSet=set;writes[i].dstBinding=i;writes[i].descriptorCount=1; - writes[i].descriptorType=(i==4 || (encode_nv12 && i==3))?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;writes[i].pImageInfo=&images[i]; - } - VkDescriptorBufferInfo grade_buffer{slot.grade_buffer,0,sizeof(VulkanGradeParameters)}; - if(!encode_nv12) { - writes[5].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[5].dstSet=set;writes[5].dstBinding=5;writes[5].descriptorCount=1; - writes[5].descriptorType=VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;writes[5].pBufferInfo=&grade_buffer; + views[5]=mask?make_view(reinterpret_cast(mask->data[0])->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT):views[0]; + std::array images{}; + std::array writes{}; + for(int i=0;i<6;++i){ + const int binding=i==5?6:i; + const bool storage=binding==4 || (encode_nv12 && binding==3); + images[i].sampler=storage?VK_NULL_HANDLE:sampler;images[i].imageView=views[i];images[i].imageLayout=VK_IMAGE_LAYOUT_GENERAL; + auto& write=writes[binding]; + write.sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET;write.dstSet=set;write.dstBinding=binding;write.descriptorCount=1; + write.descriptorType=storage?VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER;write.pImageInfo=&images[i]; } - vkUpdateDescriptorSets(device,encode_nv12?5:6,writes.data(),0,nullptr); + // Scopes count into GPU memory; every other mode reads its parameters. + VkDescriptorBufferInfo data_buffer{scope?slot.counts.buffer:slot.data.buffer,0,VK_WHOLE_SIZE}; + writes[5].sType=VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[5].dstSet=set;writes[5].dstBinding=5;writes[5].descriptorCount=1; + writes[5].descriptorType=VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;writes[5].pBufferInfo=&data_buffer; + // The NV12 encoder shader uses bindings 0-4 only. + if(encode_nv12) vkUpdateDescriptorSets(device,5,writes.data(),0,nullptr); + else vkUpdateDescriptorSets(device,7,writes.data(),0,nullptr); VkCommandBufferBeginInfo bi{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};bi.flags=VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; VkCheck(vkBeginCommandBuffer(cmd,&bi),"Begin affine commands"); std::vector semaphores; @@ -448,11 +496,31 @@ class Pipeline { } } } + auto buffer_barrier=[&](VkBuffer buffer,VkAccessFlags from,VkAccessFlags to,VkPipelineStageFlags after,VkPipelineStageFlags before){ + VkBufferMemoryBarrier barrier{VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER}; + barrier.srcAccessMask=from;barrier.dstAccessMask=to; + barrier.srcQueueFamilyIndex=barrier.dstQueueFamilyIndex=VK_QUEUE_FAMILY_IGNORED; + barrier.buffer=buffer;barrier.size=VK_WHOLE_SIZE; + vkCmdPipelineBarrier(cmd,after,before,0,0,nullptr,1,&barrier,0,nullptr); + }; + if(scope) { + vkCmdFillBuffer(cmd,slot.counts.buffer,0,count_bytes,0); + buffer_barrier(slot.counts.buffer,VK_ACCESS_TRANSFER_WRITE_BIT,VK_ACCESS_SHADER_READ_BIT|VK_ACCESS_SHADER_WRITE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); + } vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,0,0,nullptr,0,nullptr,barriers.size(),barriers.data()); vkCmdBindPipeline(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,pipeline); vkCmdBindDescriptorSets(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,layout,0,1,&set,0,nullptr); vkCmdPushConstants(cmd,layout,VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(p),&p); vkCmdDispatch(cmd,(width+15)/16,(height+15)/16,1); + if(scope) { + buffer_barrier(slot.counts.buffer,VK_ACCESS_SHADER_WRITE_BIT,VK_ACCESS_TRANSFER_READ_BIT, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,VK_PIPELINE_STAGE_TRANSFER_BIT); + VkBufferCopy region{0,0,count_bytes}; + vkCmdCopyBuffer(cmd,slot.counts.buffer,slot.staging.buffer,1,®ion); + buffer_barrier(slot.staging.buffer,VK_ACCESS_TRANSFER_WRITE_BIT,VK_ACCESS_HOST_READ_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT,VK_PIPELINE_STAGE_HOST_BIT); + } VkCheck(vkEndCommandBuffer(cmd),"End affine commands"); VkTimelineSemaphoreSubmitInfo ti{VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO}; ti.waitSemaphoreValueCount=waits.size();ti.pWaitSemaphoreValues=waits.data();ti.signalSemaphoreValueCount=signal_values.size();ti.pSignalSemaphoreValues=signal_values.data(); @@ -474,6 +542,13 @@ class Pipeline { // Publish semaphore values before releasing frame locks. Do not wait here: // later readers wait on those semaphores. locks.clear(); + if(scope) { + // Only the small counter buffer returns to the CPU, never the image. + Retire(slot); + Sync(slot.staging,false); + std::memcpy(effect->counts->data(),slot.staging.mapped,count_bytes); + return nullptr; + } return std::make_shared(output.get(),!encode_nv12); }catch(...){ // Imported surface destructors acquire the same frame mutexes. @@ -536,7 +611,16 @@ std::shared_ptr CropVulkanImage(std::shared_ptr GradeVulkanImage(std::shared_ptr source, const VulkanGradeParameters& parameters) { const int width=source->GetWidth(),height=source->GetHeight(); - EffectParameters effect{3,parameters.controls,¶meters}; + EffectParameters effect{3,parameters.controls,¶meters,sizeof(parameters)}; + return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); +} +std::shared_ptr LookupVulkanImage(std::shared_ptr source, + std::shared_ptr> table,int size,bool cube,const std::array& intensity) { + const size_t entries=cube?size_t(size)*size*size:size_t(size); + if(!table || size<=0 || table->size()!=6+entries*3) throw std::invalid_argument("Invalid Vulkan LUT"); + const int width=source->GetWidth(),height=source->GetHeight(); + EffectParameters effect{6,{float(size),cube?1.0f:0.0f,intensity[0],intensity[1],intensity[2]}, + table->data(),table->size()*sizeof(float),table}; return RenderImage(source,nullptr,width,height,QTransform(),1.0,Qt::transparent,false,&effect); } std::shared_ptr MaskVulkanImage(std::shared_ptr source, @@ -545,15 +629,46 @@ std::shared_ptr MaskVulkanImage(std::shared_ptr PremultipliedRGBA(std::shared_ptr source) { + auto rgba=std::dynamic_pointer_cast(source); + if(!rgba) throw std::invalid_argument("Vulkan effect requires a Vulkan surface"); + const auto* context=reinterpret_cast(rgba->GetFrame()->hw_frames_ctx->data); + if(context->sw_format==AV_PIX_FMT_RGBA && rgba->IsPremultiplied()) return source; + return RenderVulkanImage(source,nullptr,source->GetWidth(),source->GetHeight(),QTransform(),1.0); +} +std::shared_ptr BlendMaskVulkanImage(std::shared_ptr original, + std::shared_ptr effected,std::shared_ptr mask,bool invert) { + // GPU division is not correctly rounded; share the CPU's exact factors. + static const auto factors=[]{ + auto table=std::make_shared>(512); + for(int gray=0;gray<256;++gray) { + (*table)[gray]=static_cast(gray)/255.0f; + (*table)[256+gray]=1.0f-(*table)[gray]; + } + return std::shared_ptr>(table); + }(); + auto mask_surface=std::dynamic_pointer_cast(PremultipliedRGBA(mask)); + EffectParameters effect{7,{invert?1.0f:0.0f},factors->data(),factors->size()*sizeof(float),factors}; + effect.mask=mask_surface.get(); + return RenderImage(effected,PremultipliedRGBA(original),effected->GetWidth(),effected->GetHeight(), + QTransform(),1.0,Qt::transparent,false,&effect); +} +std::vector ScopeVulkanImage(std::shared_ptr source,const QRect& region, + int waveform_columns,int waveform_bins,int vectorscope_size) { + if(region.isEmpty() || waveform_columns<=0 || waveform_bins<256 || vectorscope_size<=0) + throw std::invalid_argument("Invalid Vulkan scope layout"); + std::vector counts(4*256+4*size_t(waveform_columns)*waveform_bins+size_t(vectorscope_size)*vectorscope_size); + EffectParameters effect{8,{float(region.left()),float(region.top()),float(region.right()+1),float(region.bottom()+1), + float(waveform_columns),float(waveform_bins),float(vectorscope_size),float(region.width())}}; + effect.counts=&counts; + RenderImage(source,nullptr,source->GetWidth(),source->GetHeight(),QTransform(),1.0,Qt::transparent,false,&effect); + return counts; +} std::shared_ptr BlurVulkanImage(std::shared_ptr source, const QRect& area,int horizontal,int vertical,int iterations) { const int width=source->GetWidth(),height=source->GetHeight(); - // Blur samples premultiplied RGBA byte values, as the CPU implementation does. - auto rgba=std::dynamic_pointer_cast(source); - if(!rgba) throw std::invalid_argument("Blur requires a Vulkan surface"); - const auto* context=reinterpret_cast(rgba->GetFrame()->hw_frames_ctx->data); - if(context->sw_format!=AV_PIX_FMT_RGBA || !rgba->IsPremultiplied()) - source=RenderVulkanImage(source,nullptr,width,height,QTransform(),1.0); + source=PremultipliedRGBA(source); auto result=source; for(int i=0;i #include #include +#include #include #include #include @@ -31,6 +32,19 @@ struct VulkanGradeParameters { }; std::shared_ptr GradeVulkanImage(std::shared_ptr source, const VulkanGradeParameters& parameters); +/// Apply a 1D or 3D .cube LUT as ColorMap does. The table holds the domain +/// minimum and maximum (3 floats each), then the RGB entries. Pass the same +/// table object each frame so the GPU copy is reused. +std::shared_ptr LookupVulkanImage(std::shared_ptr source, + std::shared_ptr> table, int size, bool cube, const std::array& intensity); +/// Blend an effect result over its input by a grayscale mask, as EffectBase does. +std::shared_ptr BlendMaskVulkanImage(std::shared_ptr original, + std::shared_ptr effected, std::shared_ptr mask, bool invert); +/// Count FrameScope video data for a region on the GPU. Returns luma, red, +/// green and blue histograms (256 each), the same four waveforms +/// (columns x bins each), then the vectorscope (size x size). +std::vector ScopeVulkanImage(std::shared_ptr source, const QRect& region, + int waveform_columns, int waveform_bins, int vectorscope_size); /// Apply a reader mask using the existing brightness/contrast and inversion semantics. std::shared_ptr MaskVulkanImage(std::shared_ptr source, std::shared_ptr mask, float contrast, int brightness, bool invert, bool replace); diff --git a/src/effects/Blur.cpp b/src/effects/Blur.cpp index 3fb21b619..cce4531f7 100644 --- a/src/effects/Blur.cpp +++ b/src/effects/Blur.cpp @@ -114,7 +114,7 @@ std::shared_ptr Blur::GetFrame(std::shared_ptr const int vertical_area_radius=std::min(std::max(0,vertical_radius_value),area.height()-1); if (!horizontal_area_radius && !vertical_area_radius) return frame; #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (VulkanRenderingEnabled() && !MaskReader() && MaskSourceId().empty()) { + if (VulkanRenderingEnabled()) { try { auto surface=frame->GetDeferredImage(); if (!surface && ParentClip() && frame->has_image_data) surface=UploadVulkanImage(*frame->GetImage()); diff --git a/src/effects/Blur.h b/src/effects/Blur.h index d9ac15b50..2be2a8d8a 100644 --- a/src/effects/Blur.h +++ b/src/effects/Blur.h @@ -39,6 +39,9 @@ namespace openshot */ class Blur : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: /// Init effect settings void init_effect_details(); diff --git a/src/effects/ColorGrade.cpp b/src/effects/ColorGrade.cpp index feea618d1..42757b23e 100644 --- a/src/effects/ColorGrade.cpp +++ b/src/effects/ColorGrade.cpp @@ -225,6 +225,15 @@ void ColorGrade::sync_lut_effect() { lut_dirty = false; } +std::shared_ptr ColorGrade::apply_lut(std::shared_ptr frame, int64_t frame_number) { + if (lut_path.empty() || lut_intensity.GetValue(frame_number) <= 0.0) + return frame; + sync_lut_effect(); + // Let the LUT upload clip images itself, as ColorGrade does. + lut_effect.ParentClip(ParentClip()); + return lut_effect.GetFrame(frame, frame_number); +} + std::shared_ptr ColorGrade::GetFrame(std::shared_ptr frame, int64_t frame_number) { const float temperature_value = std::max(-1.0f, std::min(1.0f, static_cast(temperature.GetValue(frame_number)))); const float tint_value = std::max(-1.0f, std::min(1.0f, static_cast(tint.GetValue(frame_number)))); @@ -246,11 +255,7 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptr 0.0) { - sync_lut_effect(); - return lut_effect.GetFrame(frame, frame_number); - } - return frame; + return apply_lut(frame, frame_number); } const float exposure_gain = std::pow(2.0f, exposure_value); const float contrast_factor = std::max(0.0f, 1.0f + contrast_value); @@ -264,8 +269,8 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptrGetDeferredImage(); if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); @@ -279,7 +284,7 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptrAddDeferredImage(GradeVulkanImage(surface,parameters)); - return frame; + return apply_lut(frame, frame_number); } } catch (const std::exception& error) { Logger::Instance()->Log(std::string("Vulkan grading CPU fallback: ") + error.what()); @@ -412,12 +417,7 @@ std::shared_ptr ColorGrade::GetFrame(std::shared_ptr 0.0) { - sync_lut_effect(); - frame = lut_effect.GetFrame(frame, frame_number); - } - - return frame; + return apply_lut(frame, frame_number); } std::string ColorGrade::Json() const { diff --git a/src/effects/ColorGrade.h b/src/effects/ColorGrade.h index 0649ad0d8..2ee6a6dce 100644 --- a/src/effects/ColorGrade.h +++ b/src/effects/ColorGrade.h @@ -72,6 +72,9 @@ namespace openshot */ class ColorGrade : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: ColorMap lut_effect; std::string lut_path; @@ -79,6 +82,8 @@ namespace openshot void init_effect_details(); void sync_lut_effect(); + /// Apply the optional LUT through ColorMap, on Vulkan when the frame is. + std::shared_ptr apply_lut(std::shared_ptr frame, int64_t frame_number); static float Clamp01(float value); diff --git a/src/effects/ColorMap.cpp b/src/effects/ColorMap.cpp index ef735f427..69e23984d 100644 --- a/src/effects/ColorMap.cpp +++ b/src/effects/ColorMap.cpp @@ -15,6 +15,11 @@ #include #include #include +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanCompositor.h" +#include "VulkanFrameImage.h" +#include "Logger.h" +#endif using namespace openshot; @@ -22,6 +27,7 @@ void ColorMap::load_cube_file() { if (lut_path.empty()) { lut_data.clear(); + gpu_table.reset(); lut_size = 0; lut_type = LUTType::None; needs_refresh = false; @@ -126,7 +132,12 @@ void ColorMap::load_cube_file() lut_type = parsed_is_3d ? LUTType::LUT3D : LUTType::LUT1D; lut_domain_min = parsed_domain_min; lut_domain_max = parsed_domain_max; + auto table = std::make_shared>(lut_domain_min.begin(), lut_domain_min.end()); + table->insert(table->end(), lut_domain_max.begin(), lut_domain_max.end()); + table->insert(table->end(), lut_data.begin(), lut_data.end()); + gpu_table = table; } else { + gpu_table.reset(); lut_data.clear(); lut_size = 0; lut_type = LUTType::None; @@ -186,15 +197,31 @@ ColorMap::GetFrame(std::shared_ptr frame, int64_t frame_number) if (lut_data.empty() || lut_size <= 0 || lut_type == LUTType::None) return frame; - auto image = frame->GetImage(); - int w = image->width(), h = image->height(); - unsigned char *pixels = image->bits(); - float overall = float(intensity.GetValue(frame_number)); float tR = float(intensity_r.GetValue(frame_number)) * overall; float tG = float(intensity_g.GetValue(frame_number)) * overall; float tB = float(intensity_b.GetValue(frame_number)) * overall; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && gpu_table) { + try { + auto surface = frame->GetDeferredImage(); + if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); + if (surface) { + frame->AddDeferredImage(LookupVulkanImage(surface, gpu_table, lut_size, + lut_type == LUTType::LUT3D, {tR, tG, tB})); + return frame; + } + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan LUT CPU fallback: ") + error.what()); + } + } +#endif + + auto image = frame->GetImage(); + int w = image->width(), h = image->height(); + unsigned char *pixels = image->bits(); + const bool use3d = (lut_type == LUTType::LUT3D); const bool use1d = (lut_type == LUTType::LUT1D); const int lut_dim = lut_size; diff --git a/src/effects/ColorMap.h b/src/effects/ColorMap.h index 61525fb93..7203c6f55 100644 --- a/src/effects/ColorMap.h +++ b/src/effects/ColorMap.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,9 @@ namespace openshot */ class ColorMap : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: enum class LUTType { None, LUT1D, LUT3D }; @@ -44,6 +48,7 @@ namespace openshot bool needs_refresh; ///< Reload LUT on next frame std::array lut_domain_min; ///< Input domain minimum per channel std::array lut_domain_max; ///< Input domain maximum per channel + std::shared_ptr> gpu_table; ///< Domain and entries, shared with Vulkan /// Populate info fields (class_name, name, description) void init_effect_details(); diff --git a/src/effects/Crop.cpp b/src/effects/Crop.cpp index 350e5cb16..3248844cf 100644 --- a/src/effects/Crop.cpp +++ b/src/effects/Crop.cpp @@ -99,7 +99,7 @@ std::shared_ptr Crop::GetFrame(std::shared_ptr } #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (VulkanRenderingEnabled() && !MaskReader() && MaskSourceId().empty()) { + if (VulkanRenderingEnabled()) { const QRect canvas = resize ? paint_r.toRect() : QRect(QPoint(0,0),sz); if (canvas.width()>0 && canvas.height()>0 && paint_r.width()>0 && paint_r.height()>0) { try { diff --git a/src/effects/Crop.h b/src/effects/Crop.h index 9d03a8dc9..aadd9919e 100644 --- a/src/effects/Crop.h +++ b/src/effects/Crop.h @@ -36,6 +36,9 @@ namespace openshot */ class Crop : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: /// Init effect settings void init_effect_details(); diff --git a/src/effects/Saturation.cpp b/src/effects/Saturation.cpp index 698de37a6..03b2d3eb4 100644 --- a/src/effects/Saturation.cpp +++ b/src/effects/Saturation.cpp @@ -129,7 +129,7 @@ void Saturation::init_effect_details() std::shared_ptr Saturation::GetFrame(std::shared_ptr frame, int64_t frame_number) { #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (VulkanRenderingEnabled() && !MaskReader() && MaskSourceId().empty()) { + if (VulkanRenderingEnabled()) { try { auto surface = frame->GetDeferredImage(); if (!surface && ParentClip() && frame->has_image_data) surface = UploadVulkanImage(*frame->GetImage()); diff --git a/src/effects/Saturation.h b/src/effects/Saturation.h index cea64cdf8..e5ed3df3f 100644 --- a/src/effects/Saturation.h +++ b/src/effects/Saturation.h @@ -45,6 +45,9 @@ namespace openshot */ class Saturation : public EffectBase { + protected: + bool RendersOnVulkan() const override { return true; } + private: /// Init effect settings void init_effect_details(); diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp index bf236d9a3..66e5c0326 100644 --- a/src/vulkan/affine.comp +++ b/src/vulkan/affine.comp @@ -5,7 +5,10 @@ layout(binding=1) uniform sampler2D plane_u; layout(binding=2) uniform sampler2D plane_v; layout(binding=3) uniform sampler2D background_tex; layout(binding=4, rgba8) uniform writeonly image2D output_img; -layout(std430,binding=5) readonly buffer GradingParameters { float values[]; } grading; +// Effect parameters and tables (float bits), or scope counters. +layout(std430,binding=5) buffer Data { uint data[]; }; +layout(binding=6) uniform sampler2D mask_tex; +shared uint scope_histograms[1024]; layout(push_constant) uniform Parameters { vec4 matrix_x; vec4 matrix_y; @@ -47,35 +50,107 @@ vec4 sample_source(sampler2D image, vec2 uv) { } return result/total; } +float value(int index) { return uintBitsToFloat(data[index]); } vec3 wheel_bias(int wheel) { int start=9+wheel*3; - return vec3(grading.values[start],grading.values[start+1],grading.values[start+2]); + return vec3(value(start),value(start+1),value(start+2)); } float grade_curve(int curve,float value) { int index=clamp(int(value*255.0+0.5),0,255); - return grading.values[21+curve*256+index]; + return uintBitsToFloat(data[21+curve*256+index]); +} +// Premultiplied RGBA from any supported source format. +vec4 load_source(vec2 uv) { + if(p.modes.x>=2) { + vec4 front=clamp(sample_source(plane_y,uv),0.0,1.0); + if(p.modes.x==3) front.rgb*=front.a; + else front.rgb=min(front.rgb,vec3(front.a)); + return front; + } + float y=(sample_source(plane_y,uv).r-p.extra.y)*p.color_coeffs.x; + vec2 chroma=p.modes.x==1 ? sample_source(plane_u,uv).rg : vec2(sample_source(plane_u,uv).r,sample_source(plane_v,uv).r); + chroma-=p.extra.z; + return vec4(clamp(vec3(y+p.color_coeffs.y*chroma.y, + y+p.color_coeffs.z*chroma.x+p.color_coeffs.w*chroma.y, + y+p.extra.x*chroma.x),0,1),1); +} +// ColorMap table: domain min/max, then RGB entries. +vec3 lut_entry(int index) { int i=6+index*3; return vec3(value(i),value(i+1),value(i+2)); } +float lut_1d(float v,int channel,int size) { + if(size<=1) return value(6+channel); + precise float scaled=v*float(size-1); + int i0=int(floor(scaled)),i1=min(i0+1,size-1); + precise float t=scaled-float(i0); + int last=size*3-1; + return value(6+clamp(i0*3+channel,0,last))*(1.0-t)+value(6+clamp(i1*3+channel,0,last))*t; +} +vec3 lut_3d(vec3 v,int n) { + precise vec3 f=v*float(n-1); + ivec3 i0=ivec3(floor(f)),i1=min(i0+1,ivec3(n-1)); + precise vec3 d=f-vec3(i0); + #define LUT(r,g,b) lut_entry(((b*n+g)*n+r)) + precise vec3 c00=LUT(i0.r,i0.g,i0.b)*(1.0-d.r)+LUT(i1.r,i0.g,i0.b)*d.r; + precise vec3 c01=LUT(i0.r,i0.g,i1.b)*(1.0-d.r)+LUT(i1.r,i0.g,i1.b)*d.r; + precise vec3 c10=LUT(i0.r,i1.g,i0.b)*(1.0-d.r)+LUT(i1.r,i1.g,i0.b)*d.r; + precise vec3 c11=LUT(i0.r,i1.g,i1.b)*(1.0-d.r)+LUT(i1.r,i1.g,i1.b)*d.r; + #undef LUT + precise vec3 c0=c00*(1.0-d.g)+c10*d.g; + precise vec3 c1=c01*(1.0-d.g)+c11*d.g; + return c0*(1.0-d.b)+c1*d.b; +} +// FrameScope analysis: shared histograms per workgroup, global waveforms and +// vectorscope. Matches FrameScope::analyze_video's byte math. +void scope() { + for(uint i=gl_LocalInvocationIndex;i<1024u;i+=256u) scope_histograms[i]=0u; + barrier(); + ivec2 pixel=ivec2(gl_GlobalInvocationID.xy); + ivec4 region=ivec4(p.effect); + if(all(greaterThanEqual(pixel,region.xy)) && all(lessThan(pixel,region.zw))) { + ivec4 c=ivec4(round(load_source((vec2(pixel)+0.5)/vec2(p.sizes.zw))*255.0)); + if(c.a>0) { + const float inv255=1.0/255.0; + precise vec3 f; + ivec3 index; + if(c.a==255) { f=vec3(c.rgb)*inv255; index=c.rgb; } + else { + f=min(vec3(1.0),(vec3(c.rgb)*(255.0/float(c.a)))*inv255); + index=ivec3(f*255.0+0.5); + } + precise float luma=0.299*f.r+0.587*f.g+0.114*f.b; + int luma_index=int(luma*255.0+0.5); + int columns=int(p.clear.x),bins=int(p.clear.y),size=int(p.clear.z); + int column=clamp(((pixel.x-region.x)*columns)/max(1,int(p.clear.w)),0,columns-1); + uint plane=uint(columns*bins),wave=1024u+uint(column*bins); + precise float u=-0.14713*f.r-0.28886*f.g+0.43600*f.b; + precise float v=0.61500*f.r-0.51499*f.g-0.10001*f.b; + float middle=float(size-1)*0.5; + int vx=clamp(int(middle+(u/0.43600)*middle+0.5),0,size-1); + int vy=clamp(int(middle-(v/0.61500)*middle+0.5),0,size-1); + atomicAdd(scope_histograms[luma_index],1u); + atomicAdd(scope_histograms[256+index.r],1u); + atomicAdd(scope_histograms[512+index.g],1u); + atomicAdd(scope_histograms[768+index.b],1u); + atomicAdd(data[wave+uint(luma_index)],1u); + atomicAdd(data[wave+plane+uint(index.r)],1u); + atomicAdd(data[wave+2u*plane+uint(index.g)],1u); + atomicAdd(data[wave+3u*plane+uint(index.b)],1u); + atomicAdd(data[1024u+4u*plane+uint(vy*size+vx)],1u); + } + } + barrier(); + for(uint i=gl_LocalInvocationIndex;i<1024u;i+=256u) + if(scope_histograms[i]!=0u) atomicAdd(data[i],scope_histograms[i]); } void main() { + // Scopes count instead of writing an image; every invocation reaches the barriers. + if(p.modes.w==8) { scope(); return; } ivec2 pixel=ivec2(gl_GlobalInvocationID.xy); if(any(greaterThanEqual(pixel,p.sizes.xy))) return; vec2 center=vec2(pixel)+0.5; vec2 src=vec2(dot(p.matrix_x.xyz,vec3(center,1)),dot(p.matrix_y.xyz,vec3(center,1))); vec4 front=vec4(0); if(all(greaterThanEqual(src,vec2(0))) && all(lessThan(src,vec2(p.sizes.zw)))) { - vec2 uv=src/vec2(p.sizes.zw); - if(p.modes.x>=2) { - front=sample_source(plane_y,uv); - front=clamp(front,0.0,1.0); - if(p.modes.x==3) front.rgb*=front.a; - else front.rgb=min(front.rgb,vec3(front.a)); - } else { - float y=(sample_source(plane_y,uv).r-p.extra.y)*p.color_coeffs.x; - vec2 chroma=p.modes.x==1 ? sample_source(plane_u,uv).rg : vec2(sample_source(plane_u,uv).r,sample_source(plane_v,uv).r); - chroma-=p.extra.z; - front=vec4(clamp(vec3(y+p.color_coeffs.y*chroma.y, - y+p.color_coeffs.z*chroma.x+p.color_coeffs.w*chroma.y, - y+p.extra.x*chroma.x),0,1),1); - } + front=load_source(src/vec2(p.sizes.zw)); if(p.modes.w==1 && front.a>0.0) { // Match Saturation's integer rounding and fixed-point brightness. int alpha=int(round(front.a*255.0)); @@ -115,6 +190,21 @@ void main() { rgb=clamp(mix(original,rgb,p.matrix_x.w),0.0,1.0); front.rgb=floor(rgb*front.a*255.0+0.5)/255.0; } + if(p.modes.w==6 && front.a>0.0) { + // ColorMap: straight color from premultiplied bytes, truncating writes. + ivec4 bytes=ivec4(round(front*255.0)); + float alpha=float(bytes.a)/255.0; + precise vec3 color=(vec3(bytes.rgb)/alpha)*(1.0/255.0); + vec3 low=vec3(value(0),value(1),value(2)),range=vec3(value(3),value(4),value(5))-low; + vec3 domain; + for(int i=0;i<3;++i) domain[i]=clamp(range[i]<=0.0?color[i]:(color[i]-low[i])/range[i],0.0,1.0); + int size=int(p.effect.x); + vec3 mapped=p.effect.y!=0.0?lut_3d(domain,size): + vec3(lut_1d(domain.r,0,size),lut_1d(domain.g,1,size),lut_1d(domain.b,2,size)); + vec3 amount=vec3(p.effect.zw,p.clear.x); + precise vec3 blended=(mapped*amount+color*(1.0-amount))*alpha; + front.rgb=vec3(clamp(ivec3(blended*255.0),ivec3(0),ivec3(255)))/255.0; + } front*=p.extra.w; } if(p.modes.w==2 && (any(lessThan(center,p.effect.xy)) || any(greaterThanEqual(center,p.effect.zw)))) @@ -144,6 +234,15 @@ void main() { front=floor(round(front*255.0)*float(alpha)/255.0)/255.0; } } - if(p.modes.w==3 || p.modes.w==4 || p.modes.w==5) back=vec4(0); + if(p.modes.w==7) { + // EffectBase::BlendWithMask: mix input (background) and result by mask gray. + ivec4 effected=ivec4(round(front*255.0)),original=ivec4(round(back*255.0)); + ivec4 mask=ivec4(round(texelFetch(mask_tex,pixel,0)*255.0)); + int gray=(mask.r*11+mask.g*16+mask.b*5)/32; + if(p.effect.x!=0.0) gray=255-gray; + precise vec4 mixed=vec4(original)*value(256+gray)+vec4(effected)*value(gray); + front=floor(mixed)/255.0; + } + if(p.modes.w>=3) back=vec4(0); imageStore(output_img,pixel,front+back*(1-front.a)); } diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 737a7fcec..75cb49406 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -18,6 +18,10 @@ #include "effects/Saturation.h" #include "effects/Crop.h" #include "effects/ColorGrade.h" +#include "effects/ColorMap.h" +#include +#include +#include #include "effects/Mask.h" #include "effects/Blur.h" #include "QtImageReader.h" @@ -933,3 +937,196 @@ TEST_CASE("Scope analysis preserves the cached GPU preview surface", "[vulkan][h CHECK(scope.HasVideo()); CHECK(frame->GetDeferredImage()==source); } + +namespace { +QImage EffectPixels(int width=32,int height=16) { + QImage pixels(width,height,QImage::Format_RGBA8888_Premultiplied); + for (int y=0; y> effects{{&crop,0},{&grade,1},{&saturation,2},{&lut,1}}; + for (const auto& [effect,tolerance] : effects) { + effect->MaskReader(new openshot::QtImageReader(path.toStdString())); + for (bool invert : {false,true}) { + effect->mask_invert=invert; + INFO(effect->info.class_name << " invert=" << invert); + CheckVulkanEffect(*effect,pixels,source,1,tolerance); + } + } +} + +TEST_CASE("Vulkan scopes count on the GPU like the CPU", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QImage pixels=EffectPixels(48,24); + pixels.setPixelColor(1,1,QColor(255,255,255)); + pixels.setPixelColor(2,1,QColor(0,0,0)); + pixels.setPixelColor(5,1,QColor(255,10,250)); + auto source=openshot::UploadVulkanImage(pixels); + for (bool region : {false,true}) { + for (int columns : {20,64}) { + openshot::FrameScope cpu,gpu; + for (auto* scope : {&cpu,&gpu}) { + scope->SetWaveformColumns(columns); + scope->SetVectorscopeSize(64); + if (region) scope->SetVideoRegionNormalized(0.2f,0.1f,0.5f,0.7f); + } + auto cpu_frame=std::make_shared(1,pixels.width(),pixels.height(),"transparent"); + cpu_frame->AddImage(std::make_shared(pixels.copy())); + auto gpu_frame=std::make_shared(1,pixels.width(),pixels.height(),"transparent"); + gpu_frame->AddDeferredImage(source); + cpu.SetFrame(cpu_frame); + const auto readbacks=openshot::VulkanFrameReadbackCount(); + gpu.SetFrame(gpu_frame); + INFO("region=" << region << " columns=" << columns); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + CHECK(gpu_frame->GetDeferredImage()==source); + REQUIRE(gpu.HasVideo()); + CHECK(gpu.GetVideoHistogramLumaBins()==cpu.GetVideoHistogramLumaBins()); + CHECK(gpu.GetVideoHistogramRedBins()==cpu.GetVideoHistogramRedBins()); + CHECK(gpu.GetVideoHistogramGreenBins()==cpu.GetVideoHistogramGreenBins()); + CHECK(gpu.GetVideoHistogramBlueBins()==cpu.GetVideoHistogramBlueBins()); + CHECK(gpu.GetVideoWaveformLumaBins()==cpu.GetVideoWaveformLumaBins()); + CHECK(gpu.GetVideoWaveformRedBins()==cpu.GetVideoWaveformRedBins()); + CHECK(gpu.GetVideoWaveformGreenBins()==cpu.GetVideoWaveformGreenBins()); + CHECK(gpu.GetVideoWaveformBlueBins()==cpu.GetVideoWaveformBlueBins()); + CHECK(gpu.GetVideoVectorscopeBins()==cpu.GetVideoVectorscopeBins()); + CHECK(gpu.GetVideoClippedShadows()==cpu.GetVideoClippedShadows()); + CHECK(gpu.GetVideoClippedHighlights()==cpu.GetVideoClippedHighlights()); + CHECK(gpu.GetVideoClippedRed()==cpu.GetVideoClippedRed()); + CHECK(gpu.GetVideoClippedGreen()==cpu.GetVideoClippedGreen()); + CHECK(gpu.GetVideoClippedBlue()==cpu.GetVideoClippedBlue()); + CHECK(std::abs(gpu.GetVideoAverageLuma()-cpu.GetVideoAverageLuma())<=0.5/255.0+1e-6); + } + } +} + +TEST_CASE("Masked clip grading with a LUT and scopes stays on Vulkan", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + QTemporaryDir directory; + const auto artwork=directory.filePath("artwork.png"),mask=directory.filePath("mask.png"); + REQUIRE(EffectPixels(64,64).save(artwork)); + QImage gradient(16,16,QImage::Format_RGBA8888); + for (int y=0; y<16; ++y) + for (int x=0; x<16; ++x) + gradient.setPixelColor(x,y,QColor(x*17,x*17,x*17)); + REQUIRE(gradient.save(mask)); + openshot::Clip clip(artwork.toStdString()); + openshot::ColorGrade grade; + Json::Value settings; + settings["lut_path"]=WriteCube(directory,true,5); + settings["lut_intensity"]=openshot::Keyframe(0.6).JsonValue(); + settings["exposure"]=openshot::Keyframe(0.4).JsonValue(); + grade.SetJsonValue(settings); + grade.MaskReader(new openshot::QtImageReader(mask.toStdString())); + clip.AddEffect(&grade); + openshot::Timeline timeline(128,128,openshot::Fraction(24,1),48000,2,openshot::LAYOUT_STEREO); + timeline.AddClip(&clip); + timeline.Open(); + openshot::ResetGpuMetrics(); + for (int i=1; i<=4; ++i) { + auto frame=timeline.GetFrame(i); + REQUIRE(frame->GetDeferredImage()); + openshot::FrameScope scope; + scope.SetVideoRegionNormalized(0.25f,0.25f,0.5f,0.5f); + scope.SetFrame(frame); + REQUIRE(scope.HasVideo()); + uint64_t counted=0; + for (auto bin : scope.GetVideoHistogramLumaBins()) counted+=bin; + CHECK(counted>0); + } + // A neutral grade applies only its LUT, still on the GPU. + grade.exposure=openshot::Keyframe(0.0); + for (int i=5; i<=6; ++i) { + auto frame=timeline.GetFrame(i); + REQUIRE(frame->GetDeferredImage()); + } + Json::Reader reader; + Json::Value metrics; + REQUIRE(reader.parse(openshot::GpuMetricsSnapshot(), metrics)); + // The clip and the still mask each upload once; nothing returns to the CPU. + CHECK(metrics["uploads"].asUInt64()==2); + CHECK(metrics["readbacks"].asUInt64()==0); + CHECK(metrics["render_cpu"].asUInt64()==0); + clip.RemoveEffect(&grade); + timeline.Close(); +} From 5b4620f4920d2b379ced37a3fe4bfd3624cbcb9b Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 16:03:24 -0600 Subject: [PATCH 43/50] Keep drive-amount and video masks on Vulkan Blend Blur and Saturation drive-amount masks on the GPU with the same exact factor tables as the default blend. Read video masks from the decoder's GPU frames and convert and scale them in one compositor pass, for both the Mask effect and common effect masks. Still masks keep their cached CPU scaling and single upload. Add coverage for drive-amount masks against the CPU and for 720p video masks with no frame readbacks. --- src/EffectBase.cpp | 56 +++++++++++++++++++++++++---- src/EffectBase.h | 9 +++++ src/VulkanCompositor.cpp | 32 ++++++++++++----- src/VulkanCompositor.h | 3 +- src/effects/Blur.h | 1 + src/effects/Mask.cpp | 6 ++-- src/effects/Saturation.h | 1 + src/vulkan/affine.comp | 6 ++-- tests/VulkanCompositor.cpp | 72 +++++++++++++++++++++++++++++++++++++- 9 files changed, 163 insertions(+), 23 deletions(-) diff --git a/src/EffectBase.cpp b/src/EffectBase.cpp index 1f8be85e2..863d97a7e 100644 --- a/src/EffectBase.cpp +++ b/src/EffectBase.cpp @@ -560,6 +560,44 @@ std::shared_ptr EffectBase::ResolveReaderMaskImage(int width,int height, return scaled_mask; } +std::shared_ptr EffectBase::ResolveReaderMaskSurface(int width, int height, int64_t frame_number) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (!mask_reader) + return {}; + std::shared_ptr surface; + bool still = true; + #pragma omp critical (open_effect_mask_reader) + { + try { + if (!mask_reader->IsOpen()) + mask_reader->Open(); + // Still images are scaled and uploaded once, then reused. + still = mask_reader->info.has_single_image; + if (!still) { + auto source_frame = mask_reader->GetFrame(MapMaskFrameNumber(frame_number)); + if (source_frame) + surface = source_frame->GetDeferredImage(); + } + } catch (const std::exception& e) { + Logger::Instance()->Log( + std::string("EffectBase::ResolveReaderMaskSurface unable to read mask frame: ") + e.what()); + } + } + if (surface) { + // One GPU pass converts decoder formats to RGBA and scales to the target. + const QTransform scale = QTransform::fromScale(double(width) / surface->GetWidth(), + double(height) / surface->GetHeight()); + return RenderVulkanImage(surface, nullptr, width, height, scale, 1.0, Qt::transparent, true); + } + auto image = ResolveReaderMaskImage(width, height, frame_number); + if (image && !image->isNull()) + return UploadVulkanImage(*image); +#else + (void) width; (void) height; (void) frame_number; +#endif + return {}; +} + std::shared_ptr EffectBase::TrackedObjectMask(std::shared_ptr target_image, int64_t frame_number) const { #ifdef USE_OPENCV if (!target_image || target_image->isNull() || trackedObjects.empty()) @@ -688,7 +726,8 @@ std::shared_ptr EffectBase::ProcessFrame(std::shared_ptr original_surface; - if (VulkanRenderingEnabled() && RendersOnVulkan() && !UseCustomMaskBlend(frame_number)) { + const bool drive = UseCustomMaskBlend(frame_number) && MaskDrivesAmount(frame_number); + if (VulkanRenderingEnabled() && RendersOnVulkan() && (!UseCustomMaskBlend(frame_number) || drive)) { original_surface = frame->GetDeferredImage(); // Clip images are uploaded as the effect would; the effect reuses this upload. if (!original_surface && ParentClip() && frame->has_image_data) { @@ -710,13 +749,16 @@ std::shared_ptr EffectBase::ProcessFrame(std::shared_ptr(width, height, QImage::Format_RGBA8888_Premultiplied), frame_number) - : ResolveReaderMaskImage(width, height, frame_number); - if (!mask_image || mask_image->isNull()) + std::shared_ptr mask; + if (ResolveMaskSourceEffect()) { + auto generated = GetMaskImage(std::make_shared(width, height, QImage::Format_RGBA8888_Premultiplied), frame_number); + if (generated && !generated->isNull()) + mask = UploadVulkanImage(*generated); + } else + mask = ResolveReaderMaskSurface(width, height, frame_number); + if (!mask) return output_frame; - output_frame->AddDeferredImage(BlendMaskVulkanImage( - original_surface, effected, UploadVulkanImage(*mask_image), mask_invert)); + output_frame->AddDeferredImage(BlendMaskVulkanImage(original_surface, effected, mask, mask_invert, drive)); return output_frame; } catch (const std::exception& error) { Logger::Instance()->Log(std::string("Vulkan effect mask CPU fallback: ") + error.what()); diff --git a/src/EffectBase.h b/src/EffectBase.h index de8857ad9..143c417c6 100644 --- a/src/EffectBase.h +++ b/src/EffectBase.h @@ -25,6 +25,7 @@ namespace openshot { + class FrameImage; // GPU surface; only produced in Vulkan builds class ReaderBase; /** @@ -96,6 +97,10 @@ namespace openshot /// Resolve reader artwork without requiring CPU pixels from the target frame. std::shared_ptr ResolveReaderMaskImage(int width, int height, int64_t frame_number); + /// Reader mask as a GPU surface at the target size. Video masks decoded on + /// the GPU stay there; other masks are read on the CPU and uploaded. + std::shared_ptr ResolveReaderMaskSurface(int width, int height, int64_t frame_number); + /// Optional override for effects that need custom mask behavior. virtual bool UseCustomMaskBlend(int64_t frame_number) const { return false; } @@ -109,6 +114,10 @@ namespace openshot /// Effects with a Vulkan path keep masked GPU frames on the GPU. virtual bool RendersOnVulkan() const { return false; } + /// True when the custom mask blend is the squared "drive amount" blend, + /// which the GPU applies too. + virtual bool MaskDrivesAmount(int64_t frame_number) const { return false; } + public: /// Parent effect (which properties will set this effect properties) EffectBase* parentEffect; diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 1fc405e94..81cfcbae0 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -638,18 +638,32 @@ static std::shared_ptr PremultipliedRGBA(std::shared_ptrGetWidth(),source->GetHeight(),QTransform(),1.0); } std::shared_ptr BlendMaskVulkanImage(std::shared_ptr original, - std::shared_ptr effected,std::shared_ptr mask,bool invert) { - // GPU division is not correctly rounded; share the CPU's exact factors. - static const auto factors=[]{ - auto table=std::make_shared>(512); - for(int gray=0;gray<256;++gray) { - (*table)[gray]=static_cast(gray)/255.0f; - (*table)[256+gray]=1.0f-(*table)[gray]; + std::shared_ptr effected,std::shared_ptr mask,bool invert,bool drive) { + // GPU division is not correctly rounded, so share the CPU's exact factors + // (and inverses) per mask gray value, for each blend variant. + static const auto tables=[]{ + std::array>,4> result; + for(int variant=0;variant<4;++variant) { + const bool flip=variant&1,squared=variant&2; + auto table=std::make_shared>(512); + for(int gray=0;gray<256;++gray) { + float factor; + if(squared) { + // Blur and Saturation "drive amount" mode. + factor=static_cast(gray)/255.0f; + if(flip) factor=1.0f-factor; + factor=factor*factor; + } else factor=static_cast(flip?255-gray:gray)/255.0f; + (*table)[gray]=factor; + (*table)[256+gray]=1.0f-factor; + } + result[variant]=table; } - return std::shared_ptr>(table); + return result; }(); + const auto& factors=tables[(invert?1:0)|(drive?2:0)]; auto mask_surface=std::dynamic_pointer_cast(PremultipliedRGBA(mask)); - EffectParameters effect{7,{invert?1.0f:0.0f},factors->data(),factors->size()*sizeof(float),factors}; + EffectParameters effect{7,{drive?1.0f:0.0f},factors->data(),factors->size()*sizeof(float),factors}; effect.mask=mask_surface.get(); return RenderImage(effected,PremultipliedRGBA(original),effected->GetWidth(),effected->GetHeight(), QTransform(),1.0,Qt::transparent,false,&effect); diff --git a/src/VulkanCompositor.h b/src/VulkanCompositor.h index 667bceb45..f8bfac744 100644 --- a/src/VulkanCompositor.h +++ b/src/VulkanCompositor.h @@ -38,8 +38,9 @@ std::shared_ptr GradeVulkanImage(std::shared_ptr LookupVulkanImage(std::shared_ptr source, std::shared_ptr> table, int size, bool cube, const std::array& intensity); /// Blend an effect result over its input by a grayscale mask, as EffectBase does. +/// drive selects the squared "drive amount" response, which keeps the input alpha. std::shared_ptr BlendMaskVulkanImage(std::shared_ptr original, - std::shared_ptr effected, std::shared_ptr mask, bool invert); + std::shared_ptr effected, std::shared_ptr mask, bool invert, bool drive = false); /// Count FrameScope video data for a region on the GPU. Returns luma, red, /// green and blue histograms (256 each), the same four waveforms /// (columns x bins each), then the vectorscope (size x size). diff --git a/src/effects/Blur.h b/src/effects/Blur.h index 2be2a8d8a..d08218fbe 100644 --- a/src/effects/Blur.h +++ b/src/effects/Blur.h @@ -41,6 +41,7 @@ namespace openshot { protected: bool RendersOnVulkan() const override { return true; } + bool MaskDrivesAmount(int64_t frame_number) const override { return UseCustomMaskBlend(frame_number); } private: /// Init effect settings diff --git a/src/effects/Mask.cpp b/src/effects/Mask.cpp index 21891346c..7b25dd6df 100644 --- a/src/effects/Mask.cpp +++ b/src/effects/Mask.cpp @@ -65,13 +65,13 @@ std::shared_ptr Mask::GetFrame(std::shared_ptr #ifdef OPENSHOT_ENABLE_GPU_FRAME if (VulkanRenderingEnabled() && MaskSourceId().empty()) { try { - auto mask=ResolveReaderMaskImage(frame->GetWidth(),frame->GetHeight(),frame_number); - if (!mask || mask->isNull()) return frame; + auto mask=ResolveReaderMaskSurface(frame->GetWidth(),frame->GetHeight(),frame_number); + if (!mask) return frame; auto surface=frame->GetDeferredImage(); if (!surface && ParentClip() && frame->has_image_data) surface=UploadVulkanImage(*frame->GetImage()); if (surface) { const float factor=20.0f/std::max(0.5f,20.0f-float(contrast.GetValue(frame_number))); - frame->AddDeferredImage(MaskVulkanImage(surface,UploadVulkanImage(*mask),factor, + frame->AddDeferredImage(MaskVulkanImage(surface,mask,factor, int(255*brightness.GetValue(frame_number)),mask_invert,replace_image)); return frame; } diff --git a/src/effects/Saturation.h b/src/effects/Saturation.h index e5ed3df3f..bcbf11317 100644 --- a/src/effects/Saturation.h +++ b/src/effects/Saturation.h @@ -47,6 +47,7 @@ namespace openshot { protected: bool RendersOnVulkan() const override { return true; } + bool MaskDrivesAmount(int64_t frame_number) const override { return UseCustomMaskBlend(frame_number); } private: /// Init effect settings diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp index 66e5c0326..75e4de826 100644 --- a/src/vulkan/affine.comp +++ b/src/vulkan/affine.comp @@ -235,13 +235,15 @@ void main() { } } if(p.modes.w==7) { - // EffectBase::BlendWithMask: mix input (background) and result by mask gray. + // EffectBase::BlendWithMask: mix input (background) and result by mask + // gray. The table holds the factor, then its inverse, per gray value. ivec4 effected=ivec4(round(front*255.0)),original=ivec4(round(back*255.0)); ivec4 mask=ivec4(round(texelFetch(mask_tex,pixel,0)*255.0)); int gray=(mask.r*11+mask.g*16+mask.b*5)/32; - if(p.effect.x!=0.0) gray=255-gray; precise vec4 mixed=vec4(original)*value(256+gray)+vec4(effected)*value(gray); front=floor(mixed)/255.0; + // Drive-amount masks keep the input's alpha. + if(p.effect.x!=0.0) front.a=float(original.a)/255.0; } if(p.modes.w>=3) back=vec4(0); imageStore(output_img,pixel,front+back*(1-front.a)); diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 75cb49406..634311b75 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -1024,7 +1024,16 @@ TEST_CASE("Vulkan effect masks blend on the GPU like the CPU", "[vulkan][hardwar openshot::Crop crop; crop.left=openshot::Keyframe(0.25); crop.bottom=openshot::Keyframe(0.3); - const std::vector> effects{{&crop,0},{&grade,1},{&saturation,2},{&lut,1}}; + // Saturation and Blur also have a "drive amount" mask mode. + openshot::Saturation driven_saturation; + driven_saturation.saturation=openshot::Keyframe(0.0); + driven_saturation.mask_mode=openshot::SATURATION_MASK_DRIVE_AMOUNT; + openshot::Blur driven_blur; + driven_blur.horizontal_radius=openshot::Keyframe(4.0); + driven_blur.vertical_radius=openshot::Keyframe(2.0); + driven_blur.mask_mode=openshot::BLUR_MASK_DRIVE_AMOUNT; + const std::vector> effects{{&crop,0},{&grade,1},{&saturation,2},{&lut,1}, + {&driven_saturation,2},{&driven_blur,1}}; for (const auto& [effect,tolerance] : effects) { effect->MaskReader(new openshot::QtImageReader(path.toStdString())); for (bool invert : {false,true}) { @@ -1130,3 +1139,64 @@ TEST_CASE("Masked clip grading with a LUT and scopes stays on Vulkan", "[vulkan] clip.RemoveEffect(&grade); timeline.Close(); } + +TEST_CASE("Video masks stay on Vulkan for Mask and effect masks", "[vulkan][hardware][effects]") { + if (!VulkanAvailable()) return; + // Scale a 720p decoded mask down to the frame on the GPU, for the Mask + // effect's wipe and for the common effect mask. + const QImage pixels=EffectPixels(320,180); + auto source=openshot::UploadVulkanImage(pixels); + openshot::Mask wipe; + wipe.MaskReader(new openshot::FFmpegReader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4")); + openshot::ColorGrade grade; + grade.exposure=openshot::Keyframe(1.0); + grade.MaskReader(new openshot::FFmpegReader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4")); + // These frames have detailed artwork; the trailer opens on black. + const std::vector numbers{300,700}; + { + openshot::FFmpegReader probe(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + probe.Open(); + for (int number : numbers) { + auto frame=probe.GetFrame(number); + REQUIRE(frame->GetDeferredImage()); + const auto image=frame->GetImage(); + int lowest=255,highest=0; + for (int y=0; yheight(); y+=8) + for (int x=0; xwidth(); x+=8) { + const int gray=qGray(image->pixel(x,y)); + lowest=std::min(lowest,gray); highest=std::max(highest,gray); + } + REQUIRE(highest-lowest>100); + } + } + for (auto* effect : std::initializer_list{&wipe,&grade}) { + for (int number : numbers) { + // GPU first: the CPU run reads the reader's cached mask frames back. + auto gpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + gpu->AddDeferredImage(source); + const auto readbacks=openshot::VulkanFrameReadbackCount(); + effect->ProcessFrame(gpu,number); + INFO(effect->info.class_name << " frame=" << number); + REQUIRE(gpu->GetDeferredImage()); + CHECK(openshot::VulkanFrameReadbackCount()==readbacks); + auto cpu=std::make_shared(number,pixels.width(),pixels.height(),"transparent"); + cpu->AddImage(std::make_shared(pixels.copy())); + ProcessCpuEffect(*effect,cpu,number); + // The GPU scales the mask with its own filter, so compare on average. + auto expected=cpu->GetImage(),actual=gpu->GetImage(); + double total=0.0; + for (int y=0; yheight(); ++y) + for (int x=0; xwidth()*4; ++x) + total+=std::abs(int(expected->constScanLine(y)[x])-int(actual->constScanLine(y)[x])); + const double mean=total/(actual->width()*actual->height()*4.0); + double changed=0.0; + for (int y=0; yheight(); ++y) + for (int x=0; xwidth()*4; ++x) + changed+=std::abs(int(pixels.constScanLine(y)[x])-int(actual->constScanLine(y)[x])); + changed/=actual->width()*actual->height()*4.0; + INFO("mean byte error=" << mean << " mean change=" << changed); + CHECK(changed>0.5); + CHECK(mean<=2.0); + } + } +} From 8b553b9de1aa2d8d7a94eed47823405bb57dabb2 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 17:38:54 -0600 Subject: [PATCH 44/50] Allow one byte in the masked blend test on Metal MoltenVK compiles shaders with fast math, which may fuse the mask blend's multiply-add and round one byte differently. Linux and Windows keep the byte-exact check. --- tests/VulkanCompositor.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 634311b75..5171db1ff 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -1020,7 +1020,14 @@ TEST_CASE("Vulkan effect masks blend on the GPU like the CPU", "[vulkan][hardwar saturation.saturation=openshot::Keyframe(0.0); QTemporaryDir tables; openshot::ColorMap lut(WriteCube(tables,true,5)); - // Crop is byte exact on its own, so it checks the blend exactly. + // Crop is byte exact on its own, so it checks the blend exactly. Metal + // (MoltenVK) compiles with fast math, which may fuse the blend's + // multiply-add and round one byte differently. +#ifdef __APPLE__ + const int blend_tolerance=1; +#else + const int blend_tolerance=0; +#endif openshot::Crop crop; crop.left=openshot::Keyframe(0.25); crop.bottom=openshot::Keyframe(0.3); @@ -1032,7 +1039,7 @@ TEST_CASE("Vulkan effect masks blend on the GPU like the CPU", "[vulkan][hardwar driven_blur.horizontal_radius=openshot::Keyframe(4.0); driven_blur.vertical_radius=openshot::Keyframe(2.0); driven_blur.mask_mode=openshot::BLUR_MASK_DRIVE_AMOUNT; - const std::vector> effects{{&crop,0},{&grade,1},{&saturation,2},{&lut,1}, + const std::vector> effects{{&crop,blend_tolerance},{&grade,1},{&saturation,2},{&lut,1}, {&driven_saturation,2},{&driven_blur,1}}; for (const auto& [effect,tolerance] : effects) { effect->MaskReader(new openshot::QtImageReader(path.toStdString())); From ae9a16e185a641abaa48034d353379034a4417a4 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 18:47:27 -0600 Subject: [PATCH 45/50] Decode 10-bit, HDR and tagged color correctly; tag exports - Decode with the source's tagged YUV matrix (BT.601 when untagged, as earlier OpenShot exports were written) on CPU and Vulkan. - Convert HLG and PQ (BT.2020) sources to SDR BT.709 with a shared, hue-preserving tone curve (BT.2408 reference white at 203 nits). CPU uses a LUT-based converter; Vulkan converts in affine.comp. - Read P010/P012/P016 and P210/P212/P216 frames directly on Vulkan, including 10-bit VideoToolbox surfaces on macOS. - Tag exports with BT.709 (HD) or BT.601 (SD) and encode with the matching matrix, on CPU and Vulkan. - Add SDR/HLG/PQ color chart fixtures and tests comparing CPU, Vulkan and expected values. --- examples/color-chart-hlg.mp4 | Bin 0 -> 3930 bytes examples/color-chart-pq.mp4 | Bin 0 -> 3809 bytes examples/color-chart-sdr-bt709.mp4 | Bin 0 -> 2352 bytes src/CMakeLists.txt | 1 + src/ColorManagement.cpp | 255 +++++++++++++++++++++++++++++ src/ColorManagement.h | 87 ++++++++++ src/FFmpegReader.cpp | 62 ++++++- src/FFmpegReader.h | 6 + src/FFmpegWriter.cpp | 27 +++ src/MacVulkanInterop.h | 6 + src/MacVulkanInterop.mm | 89 +++++++--- src/VulkanCompositor.cpp | 70 ++++++-- src/VulkanCompositor.h | 6 + src/VulkanFrameImage.cpp | 14 +- src/VulkanFrameImage.h | 6 +- src/vulkan/affine.comp | 57 ++++++- src/vulkan/encode_nv12.comp | 11 +- tests/CMakeLists.txt | 5 + tests/ColorManagement.cpp | 222 +++++++++++++++++++++++++ tests/VulkanCompositor.cpp | 189 ++++++++++++++++++++- tests/generate-color-charts.py | 85 ++++++++++ 21 files changed, 1127 insertions(+), 71 deletions(-) create mode 100644 examples/color-chart-hlg.mp4 create mode 100644 examples/color-chart-pq.mp4 create mode 100644 examples/color-chart-sdr-bt709.mp4 create mode 100644 src/ColorManagement.cpp create mode 100644 src/ColorManagement.h create mode 100644 tests/ColorManagement.cpp create mode 100644 tests/generate-color-charts.py diff --git a/examples/color-chart-hlg.mp4 b/examples/color-chart-hlg.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3f20b43750151a4279a32154972e2b0d9d540533 GIT binary patch literal 3930 zcmZu!3v?9a6`oBZkVt_5N+7`uCP08pvb%Yh#H<8^5)^_J4U|V^c4l^W$h?VO? zBEC+G2nh5Lj)}pe(ojA4z*dFw5CKmG8&R z(u9s%8mC>Xe0fS&d(lf>yTe-K@x^zvu4-?4eR}$~A8-HN=8MTSsZ-~7EZ+KF=b3ND zPW)<3e&p*zBa&8bJ3g~vf9HTDTkqPn_@Sz+xAg4MFEB-?UKzc*>9w9=j~B+B{eAtk zjQ#biTS_Y5Z620tHoq5cX*ZX2e|ll|q66}{=T@C}O&E9Z_~FWlJ@Yn)e?GK)rP(s2 z?C`QtNnbyZmb_vAl1Yynl%N_0tVw?#MX)+10f2lV?c&_dZ$EsidF?B0KJDD*y~>XH zZU4Hnc@?{^OTFtq0}3zyHS<(>#0~nK?>3)4a%N6yB=@oNZ>{b;x$MsMAAWZ8-|}`p z+&gzcQ=Ml^#mLhQWM0-Y;_K7(>D%!^!VM%7D)d*=rKzUhhH(uYocH2a&+w_n=NoZnG4V`0O=TK}$b zo7OE%sCxMOsf+GD+Vn}sA@l8~?Z57xHs?V5hVfEkuw~fiYQr;Y-tC&$GW82rN@t6_ z>&0XD|G0X=pqpKpL*Gg6`ry#qtgf<-E9RpsxPR`>d|=Jl>05f4$tM|Rx6vlNrbI%ilI(L%Y zw>Pf4H1oHM-khB_cF-nk;rMY9u6L@3-R50TCG&L;tR=Umze~hnHRB{C`j! zzB$k``Dz?4va0QG#`j~4X}iuAaKuhx%-7*%l!dN zw$lD)W{S>PnrKRthc0?PIBY7c11EkvRZZaJvE^ zedu!CG>eZ2g`ONwagN9BVHMe5=PfL74()eJZ!hNSCTJ~p#jzl*waKMFGTl8_t}x8d zgjRefrOY+h)!R!M;wcAG70;Az+?2QlhuL?!Y|(xvtx1ChkB~NQp6Px2PZet`Y%%(Z z-}gKO22OYFGk5sQCNi!;V=7%Y^hVcB+0UO!d~?){6FVP#;cCOX-IdfL2*t{HXj%SL8^A`w3g?wSY-|v@77S58jaNXjCGs?@^;+(u3_u_?x`K({I7PB5V zHpo@vKv!RpP9WDe)rl6|l92!D>2ZsQ9V_ zI2+MSiV`BLtfB`wMT$s@&rPisCBq8&JZ@IjjIhNC#PE68fI)nD?p#llXi7l6?mADg z7Zgh}6<&}aZs``UaJCTysI8TgMFDUKzH*idwciGjOj`swBXB|1B+=nuf$Cb&K{$f(ZiQqm zXxptyW>7-b9h2ZSK}Fyj4o(Y|u&M+a3Hz~oftb7ok%HUUzSOD_n1k%A5?d?LF0xSv zbjm~HmHK58N01*w76>a!enl6meeMz@gx9GwHoN4aM~Do}B9ekY%Mpn3x-XxVcvGq8 zZ2wIs3OX0yi4}D;795*Kl_F<~PKY8Af|{{YYkv#bwoEaqNJL5UidB#B8Kj;@-%F$K zzl9+1{4(nG>9b`@P$~s zMYvj1wrI0D7@ZVAWW(e!Q6P#UHT*Fo&&~2R98C%m0SI}Ov>=@i)LzsyuEy*#U6ata zVf2+<4q%c6Ob)-v)&!jjMv=k; z{khZasbFEIIn{u#>xym~rmXskSyZ1!0(7_(IdqkW5p_5bLZ+_rNImq3GMbLB=W1n5 z)N4Ty6f|SD+io@`tnwhy*0Mp5+lwOAgI|$={Ak5USjHTK*|fYdcqoo37A+q8Z&bG8P!S4~_`P2Mo*6~1p}%g%i^v(3_} X!ZiG2OqEcB7~J^+xbxkHE1&y+K&kf} literal 0 HcmV?d00001 diff --git a/examples/color-chart-pq.mp4 b/examples/color-chart-pq.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d7045f9822298b064f2f89627a332c454dc902c4 GIT binary patch literal 3809 zcmZu!3v?9K8J`GUq0OTbhqz^&FEOLW(oOE z`L?J3L7zDO)$OJGADw*W59i-5=zsX0Q@3`s-xD1$dTHL?A>Hid4}Ljr&)mb?uQ}a% zqwo0k6;IZV8r-mc%gc9ee0~0N)3Q&UIKSnA=U3C$c3$&g$F@@?>ux!mvv$R`J9^#m z(}|fw^A|t0?Z8{3rypShc=+N0^~R1RPc3}y#Z5z=eEPe`=Z}B?!qF4e z`+Pm$?EZe%N7IW#IY;U@8~f&t9M_oJ{#ozyH;rxiylKNII`3<>Z_$ih3-?@D_2&zJ zZ9Px#?D?BTHT%b3*#5=>-=cLZvohXTKe+VoU(6izZPl>@HIMzNj2+s%b8X)rS|^Oi zI{E244G&)Z)FZ(iA2)6LVRlC5zwT<-`po86r)?>{a$d#OibbE1mg&tqXC@wvww(K5 z#(hs5IMnmqgCArsednm8WzTLZerQX-s)y=)rP((P%k&Ri*m7ppl-rHFw^qNaJQMp` z9Q*ybyIv?BaOK423*COZSbhK4-mJE)eFYnrX2`978D;NlT)wsW2lM&te?Ky={K&*F zcQmdXdv?shiKVC6!(Z)MIBalvtm$v}x2*H!wk^GH(10yWAGy1xa^AjK=YBJ8 zRh#eBvSH&Z?A>in%aAEYXBT{2c+LH5_DtH+=SXdC!HWa4UaeSv{j}fzv3>TEsg*@_ zBgU=yX2Z+-Z%-}?wtw-vgA;BX`^nvH##(2^=hxk~W8w9+S?{4v#SGuUzIQ$lT*!uu znc4REsJ&MWK6%^G;VloGs(AB;y$k#oz%xMb{dee|#o29duycJH7nBYcu9(t#Z{zMC z&j>=#j|@taVA73bT*cmHWqtuf6?_8!yR?3RfP5KVaf$Q)K^gdFcgL2iaX8zlcE6d~ zg)yz|GF!lrK1nlQhF5SF{@RO25QY-fVc7GC&d|W_;u6r?Ev2i+d_&b_&_qMM-1jjc zUm@J_x+^EeRo!+Wypo!V!EkBK08Xo9Q`HIQ=`qsK%qW2VNyAmz)y$!)cb2Exg!Ayl z&wC|oUG(6bFPRh31tF(^IV>Fn@Z?+{Pbz#e@zqtQN2q|?RRH!TBI&iQtTbN@7#_KjJeg&TItIWL27o%W6lWeKmYWTAoT9G0pH23@b&U_bZ~}D z&Vf{A3S(BR&e@2={P8|lw9Cnc9@)M6krituhjwkPT2SSR@mJ<6n;_7Ag6|dk*+|8& z1z+~SX}+wE)Vex<)3KcAel_XHOLuI(IRDl5Y1~$EcqF0esu&zyHbRudLVw5~92Jxz z#Y$<3Tv8N?L`eCpshXLnn=@-tWu;i=FZ2iI%ql4oBbqZu30cP z8Z}wDs8|V24F|CTg=J#YmJK2~ahu303m1#EmL-~0vJ8CH1H6ryHb)uL3{j^sNhe97 zhXdSdl~_(Z91Mt>X(b##{;1?odb8CD#ZB z)YU3!sscC;UnNH)Hnb(;suWQ{Ll587PUQJU|vDS(PpD146!JLsL_yQbd!vkG9f9Gz6d1dL*W63gQ$HnLv10^T_e_G<3>ATIcsMUq zCK?1ZmWW^v0kLHhBE_S{&eR%7n1k#aM64yei(J$Lz49<+y}p4+4EeD%g^4PO=v1i= z2g;EUnR01tZpo#N5E+=oh>k#ONr=icTqF|N*6StLf7^=!m7+3pQjVsBfG z>4Y)VjGJ2bTgbL!t0_e)N>bLHdW6qn^*s6^9(@m{HJ!PD00M-#auuCwl}L?6qljBT zRP0XBR>F!M1tl5|`U7H(LJbRwN&s5SAr^dLj_fe0*47-}tRBWE1rW`$WlR)^qDZYs z8Yv8ja*f23fRYDYuBEWF(37@VcT&G_NA_Q~q;D-KpSUrg_zXuT!1cmaQ4#G7;5h zvM3)eB_7?7VMHBHl(2)usFz)ZfK$!!4EqU{UeT@y3>7nkQ18J=AMi9#w0jp@O5RN||!%q+0Lnn~Db@Nq=aSQU(Mot>Fwhn*KQ!(%;& z9E=IoiY-bG-H5SJOno*nMy*&6peLH@F({-FQ+(_(VG&J}Bi21cr2pNONId;_@BLrj z|KG=d_uj=Y40q}th1X;e!;Bc>(9CoRLyAf&iD6izQ{^}e3*RHMv<{eq5#htbd;k4v z-#@z_+*8X>;|+ucr4mQUIG=@7*67tVx3AQ&@@#VZK=v>E(li{`;W^-ewCpZ)rpG^ z*jKbHP*6eeBD5BYFj+uUP~IS?%P31G7&M(zs8pO+phj>WAOcd+MNocJOso{$iVMIdq9@oU*viColT?ZDHvj8X7+T zZ*Svg8_Tzw+`X;!eQ{rw95aMG*8Z>6empczvfA(dL?7I8R)K!1-V{7(|zuFRcO@6-f z3v2d?%dd3o8$P!sclnmTbzpVU@VbvX>$4lDZ4K1*f4a<4`*L#L<^1=D#^2cC{cu6EH~VATq-A^K=4bqK zLfF82@tb{J`wg3R9vz52RXFx;R>Ik{_sls>(RH=I9@y1fU4MS|{hYlzcD{7c4%8`le4<5cPhnJ!~`Zu4X^o{SHAVLw+H6b zU-G#cuJqiQ{X#g^RN-B_YB(^ctyeRS)^D0vUbz3%Lt}nt1-{tSX>CiGGk41N?gw*U z>8T~|y>K$dxs$pu({^{_svPQN$G{h=#I@%^3dS2RR zrw9B#x4XS5yCXMxO@GIRmF2hYw6&kRC4I2=?|z}G)zmkyWw&zJmh}6uysM)5x7X6A zxGTwYOl{hpD^_Crm)PJwj)5KCMv z@KH3z4+@JJF!X&wKT*K^l%Dkj`Try%{J^%t58DmS)w(-4=9p2yk!(-R1rQ^i5%5!b z7J{Iz6&i*maICHYkLLspUZEhN2*?}uIF~V8WO*8FxX3=e_7}k%fh|81j7xH{f*O>d zL+{IojQ9%l-YldgR^SlE@F83|P7q>C#9-3V(@bDPzIYa{H53el{87cKfQMuQ+)xey zug2V;hf!@N9GuTfa9tNHc zsCfn!iypp-{h|Q)fo_}4TWB;IE8%cp(EW1k=GgNFEW*`)qabn1P#}P!Co!_1N{mnj zQ$Zh~Sh$Bn8*OM?iwr=MH9c4aBqIR&(5Xt;tIH^1Jp9!`Nl;P@cHRj)pAH*O{10~!*cbo+ literal 0 HcmV?d00001 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1b293b77e..406308d92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,6 +60,7 @@ set(OPENSHOT_SOURCES ChunkReader.cpp ChunkWriter.cpp Color.cpp + ColorManagement.cpp Clip.cpp ClipBase.cpp Coordinate.cpp diff --git a/src/ColorManagement.cpp b/src/ColorManagement.cpp new file mode 100644 index 000000000..cfb441627 --- /dev/null +++ b/src/ColorManagement.cpp @@ -0,0 +1,255 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "ColorManagement.h" + +#include +#include +#include + +extern "C" { +#include +#include +#include +} + +namespace openshot { +namespace color { +namespace { + +// BT.2100 PQ (SMPTE ST 2084) +constexpr double kPqM1 = 2610.0 / 16384.0, kPqM2 = 2523.0 / 4096.0 * 128.0; +constexpr double kPqC1 = 3424.0 / 4096.0, kPqC2 = 2413.0 / 4096.0 * 32.0, kPqC3 = 2392.0 / 4096.0 * 32.0; +// BT.2100 HLG +constexpr double kHlgA = 0.17883277, kHlgB = 0.28466892, kHlgC = 0.55991073; +constexpr double kHlgGamma = 1.2; // system gamma at the nominal 1000 nit peak +// SDR display EOTF (BT.1886), as BT.2408 uses to place SDR within HDR. +constexpr double kSdrGamma = 2.4; + +// Linear BT.2020 RGB to linear BT.709 RGB (BT.2087). +constexpr float kBt2020To709[9] = { + 1.6604910f, -0.5876411f, -0.0728499f, + -0.1245505f, 1.1328999f, -0.0083494f, + -0.0181508f, -0.1005789f, 1.1187297f}; +constexpr float kLuma2020[3] = {0.2627f, 0.6780f, 0.0593f}; +constexpr float kLuma709[3] = {0.2126f, 0.7152f, 0.0722f}; + +// Lookups stay small enough for L1 cache. 12-bit signal indices exceed the +// 10-bit sources' precision (and PQ's 12-bit visibility threshold). +constexpr int kSignalBits = 12; +constexpr int kSignalSize = 1 << kSignalBits; +// Encode lookup indexed by the float's exponent and top 10 mantissa bits, +// covering linear 2^-24 (below half an 8-bit code) to 1.0. +constexpr uint32_t kEncodeShift = 13; +constexpr uint32_t kEncodeBase = (127u - 24u) << (23 - kEncodeShift); +constexpr int kEncodeSize = int((127u << (23 - kEncodeShift)) - kEncodeBase) + 1; + +// Tone scale lookup indexed by the brightest channel's exponent and top 7 +// mantissa bits (under 0.4% apart), from the knee to 64x reference white. +constexpr uint32_t kToneShift = 16; +constexpr uint32_t kToneBase = (127u - 1u) << (23 - kToneShift); +constexpr int kToneSize = int(((127u + 6u) << (23 - kToneShift)) - kToneBase); + +inline uint32_t FloatBits(float value) { uint32_t bits; std::memcpy(&bits, &value, sizeof(bits)); return bits; } +inline int ToneIndex(float brightest) { + const int index = int(FloatBits(brightest) >> kToneShift) - int(kToneBase); + return std::clamp(index, 0, kToneSize - 1); +} +inline int EncodeIndex(float linear) { + const int index = int(FloatBits(linear) >> kEncodeShift) - int(kEncodeBase); + return std::clamp(index, 0, kEncodeSize - 1); +} + +double PqToNits(double e) { + const double p = std::pow(std::clamp(e, 0.0, 1.0), 1.0 / kPqM2); + return 10000.0 * std::pow(std::max(p - kPqC1, 0.0) / (kPqC2 - kPqC3 * p), 1.0 / kPqM1); +} +double HlgToScene(double e) { + e = std::clamp(e, 0.0, 1.0); + return e <= 0.5 ? e * e / 3.0 : (std::exp((e - kHlgC) / kHlgA) + kHlgB) / 12.0; +} +bool IsBt2020Matrix(AVColorSpace matrix) { + return matrix == AVCOL_SPC_BT2020_NCL || matrix == AVCOL_SPC_BT2020_CL; +} +} // namespace + +AVColorSpace DecodeMatrix(AVColorSpace tagged, int width, int height) { + switch (tagged) { + case AVCOL_SPC_BT709: + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + case AVCOL_SPC_SMPTE240M: + case AVCOL_SPC_FCC: + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: + return tagged; + default: + // Untagged files keep BT.601 at every size: earlier OpenShot exports + // and older FFmpeg encodes used it without tagging, and + // cameras and phones tag their files. + (void) width; (void) height; + return AVCOL_SPC_SMPTE170M; + } +} + +AVColorSpace EncodeMatrix(int width, int height) { + // Tagged; this is also what players assume for untagged files. + return (width >= 1280 || height > 576) ? AVCOL_SPC_BT709 : AVCOL_SPC_SMPTE170M; +} + +int SwsColorspace(AVColorSpace matrix) { + switch (matrix) { + case AVCOL_SPC_BT709: return SWS_CS_ITU709; + case AVCOL_SPC_FCC: return SWS_CS_FCC; + case AVCOL_SPC_SMPTE240M: return SWS_CS_SMPTE240M; + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: return SWS_CS_BT2020; + default: return SWS_CS_ITU601; + } +} + +float SourceColor::RelativePeak() const { + if (transfer == Transfer::Sdr) return 1.0f; + const double nits = transfer == Transfer::HLG ? kDefaultPeakNits : peak_nits; + return float(std::clamp(nits / kReferenceWhiteNits, 1.0, 10000.0 / kReferenceWhiteNits)); +} + +double FramePeakNits(const AVFrame* frame) { + if (!frame) return 0.0; + if (const AVFrameSideData* light = av_frame_get_side_data(frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL)) { + const auto* cll = reinterpret_cast(light->data); + if (cll->MaxCLL > 0) return double(cll->MaxCLL); + } + if (const AVFrameSideData* display = av_frame_get_side_data(frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA)) { + const auto* mastering = reinterpret_cast(display->data); + if (mastering->has_luminance && mastering->max_luminance.den > 0) { + const double nits = av_q2d(mastering->max_luminance); + if (nits > 0.0) return nits; + } + } + return 0.0; +} + +SourceColor DescribeFrame(const AVFrame* frame, int width, int height, double peak_nits) { + SourceColor result; + const AVColorSpace tagged = frame ? frame->colorspace : AVCOL_SPC_UNSPECIFIED; + result.matrix = DecodeMatrix(tagged, width, height); + result.full_range = frame && frame->color_range == AVCOL_RANGE_JPEG; + if (frame) { + if (frame->color_trc == AVCOL_TRC_SMPTE2084) result.transfer = Transfer::PQ; + else if (frame->color_trc == AVCOL_TRC_ARIB_STD_B67) result.transfer = Transfer::HLG; + // Unspecified primaries follow a BT.2020 matrix, as players do. + result.bt2020 = frame->color_primaries == AVCOL_PRI_BT2020 || + (frame->color_primaries == AVCOL_PRI_UNSPECIFIED && IsBt2020Matrix(tagged)); + } + if (peak_nits > 0.0) result.peak_nits = peak_nits; + return result; +} + +float ToneMap(float value, float relative_peak) { + if (value <= kToneKnee || relative_peak <= 1.0f) return std::min(value, 1.0f); + // Linear below the knee; above it an extended-Reinhard shoulder with unit + // slope at the knee that reaches 1.0 exactly at the content peak. + const float span = 1.0f - kToneKnee; + const float t = (value - kToneKnee) / span; + const float w = std::max((relative_peak - kToneKnee) / span, 1.0f); + const float shoulder = t * (1.0f + t / (w * w)) / (1.0f + t); + return std::min(kToneKnee + span * shoulder, 1.0f); +} + +SdrConverter::SdrConverter(const SourceColor& source_color) + : source(source_color), peak(source_color.RelativePeak()), linearize(kSignalSize), encode(kEncodeSize) { + for (int i = 0; i < kSignalSize; ++i) { + // Bucket centre of the 16-bit codes sharing this index. + const double e = (i + 0.5) / kSignalSize; + double value; + switch (source.transfer) { + case Transfer::PQ: value = PqToNits(e) / kReferenceWhiteNits; break; + case Transfer::HLG: value = HlgToScene(e); break; + default: value = std::pow(e, kSdrGamma); break; + } + linearize[i] = float(value); + } + if (source.transfer == Transfer::HLG) { + // OOTF: display = peak * Ys^(gamma-1) * scene, relative to reference white. + hlg_gain.resize(kSignalSize + 1); + for (int i = 0; i <= kSignalSize; ++i) + hlg_gain[i] = float(kDefaultPeakNits / kReferenceWhiteNits * std::pow(double(i) / kSignalSize, kHlgGamma - 1.0)); + } + tone_scale.resize(kToneSize); + for (int i = 0; i < kToneSize; ++i) { + const uint32_t bits = ((uint32_t(i) + kToneBase) << kToneShift) | (1u << (kToneShift - 1)); + float brightest; + std::memcpy(&brightest, &bits, sizeof(brightest)); + tone_scale[i] = brightest > kToneKnee ? ToneMap(brightest, peak) / brightest : 1.0f; + } + for (int i = 0; i < kEncodeSize; ++i) { + // Middle of the floats sharing this index. + const uint32_t bits = ((uint32_t(i) + kEncodeBase) << kEncodeShift) | (1u << (kEncodeShift - 1)); + float linear; + std::memcpy(&linear, &bits, sizeof(linear)); + encode[i] = uint8_t(std::lround(255.0 * std::pow(std::min(double(linear), 1.0), 1.0 / kSdrGamma))); + } +} + +void SdrConverter::ToSdr(float rgb[3]) const { + if (source.bt2020) { + const float r = rgb[0], g = rgb[1], b = rgb[2]; + rgb[0] = kBt2020To709[0] * r + kBt2020To709[1] * g + kBt2020To709[2] * b; + rgb[1] = kBt2020To709[3] * r + kBt2020To709[4] * g + kBt2020To709[5] * b; + rgb[2] = kBt2020To709[6] * r + kBt2020To709[7] * g + kBt2020To709[8] * b; + } + // Out-of-gamut colors clip per channel. + for (int c = 0; c < 3; ++c) rgb[c] = std::max(rgb[c], 0.0f); + const float brightest = std::max(rgb[0], std::max(rgb[1], rgb[2])); + if (brightest > kToneKnee) { + // Scale all channels by the max-channel curve, preserving hue. + const float scale = ToneMap(brightest, peak) / brightest; + for (int c = 0; c < 3; ++c) rgb[c] *= scale; + } + for (int c = 0; c < 3; ++c) rgb[c] = std::min(rgb[c], 1.0f); +} + +template +void SdrConverter::ConvertRows(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const { + const float* luma = Gamut ? kLuma2020 : kLuma709; + constexpr int shift = 16 - kSignalBits; + #pragma omp parallel for if(int64_t(width) * height >= 65536) schedule(static) + for (int y = 0; y < height; ++y) { + const uint16_t* in = reinterpret_cast(reinterpret_cast(src) + int64_t(y) * src_stride_bytes); + uint8_t* out = dst + int64_t(y) * dst_stride_bytes; + for (int x = 0; x < width; ++x, in += 4, out += 4) { + float r = linearize[in[0] >> shift], g = linearize[in[1] >> shift], b = linearize[in[2] >> shift]; + if (Hlg) { + const float scene_luma = luma[0] * r + luma[1] * g + luma[2] * b; + const float gain = hlg_gain[std::min(kSignalSize, int(scene_luma * kSignalSize + 0.5f))]; + r *= gain; g *= gain; b *= gain; + } + if (Gamut) { + const float r2 = kBt2020To709[0] * r + kBt2020To709[1] * g + kBt2020To709[2] * b; + const float g2 = kBt2020To709[3] * r + kBt2020To709[4] * g + kBt2020To709[5] * b; + const float b2 = kBt2020To709[6] * r + kBt2020To709[7] * g + kBt2020To709[8] * b; + r = r2; g = g2; b = b2; + } + // Clip out-of-gamut colors, then scale by the max-channel tone curve. + r = std::max(r, 0.0f); g = std::max(g, 0.0f); b = std::max(b, 0.0f); + const float scale = tone_scale[ToneIndex(std::max(r, std::max(g, b)))]; + out[0] = encode[EncodeIndex(r * scale)]; + out[1] = encode[EncodeIndex(g * scale)]; + out[2] = encode[EncodeIndex(b * scale)]; + out[3] = uint8_t(in[3] >> 8); + } + } +} + +void SdrConverter::Convert(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const { + const bool hlg = source.transfer == Transfer::HLG; + if (hlg && source.bt2020) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); + else if (hlg) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); + else if (source.bt2020) ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); + else ConvertRows(src, src_stride_bytes, dst, dst_stride_bytes, width, height); +} + +} // namespace color +} // namespace openshot diff --git a/src/ColorManagement.h b/src/ColorManagement.h new file mode 100644 index 000000000..dcd0a63f2 --- /dev/null +++ b/src/ColorManagement.h @@ -0,0 +1,87 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#ifndef OPENSHOT_COLOR_MANAGEMENT_H +#define OPENSHOT_COLOR_MANAGEMENT_H + +#include +#include + +extern "C" { +#include +} +struct AVFrame; + +namespace openshot { +namespace color { + +/// SDR graphics/reference white in HDR signals (ITU-R BT.2408). HDR sources +/// map this luminance to SDR 100%. +constexpr double kReferenceWhiteNits = 203.0; +/// Nominal HLG display peak (BT.2100) and the PQ peak assumed without metadata. +constexpr double kDefaultPeakNits = 1000.0; +/// Linear SDR level (1.0 = reference white) above which highlights compress. +/// Must match TONE_KNEE in src/vulkan/affine.comp. +constexpr float kToneKnee = 0.8f; + +enum class Transfer : int { Sdr = 0, PQ = 1, HLG = 2 }; + +/// The YUV matrix to decode with: the tagged matrix, or BT.601 when untagged +/// (how OpenShot exports before 2026 and older FFmpeg encodes were written). +AVColorSpace DecodeMatrix(AVColorSpace tagged, int width, int height); +/// The YUV matrix OpenShot encodes with (and tags) for an output size: +/// BT.709 for HD and larger, BT.601 for SD. +AVColorSpace EncodeMatrix(int width, int height); +/// The swscale coefficient table index (SWS_CS_*) for a matrix. +int SwsColorspace(AVColorSpace matrix); + +/// How a decoded source must be converted into OpenShot's SDR BT.709 RGBA. +struct SourceColor { + AVColorSpace matrix = AVCOL_SPC_BT709; + bool full_range = false; + Transfer transfer = Transfer::Sdr; + bool bt2020 = false; ///< BT.2020 primaries: convert gamut to BT.709 + double peak_nits = kDefaultPeakNits; ///< HDR content peak, for tone mapping + + /// True when transfer or gamut conversion is required (HDR or wide gamut). + bool NeedsConversion() const { return transfer != Transfer::Sdr || bt2020; } + /// Peak in units of reference white (>= 1); 1 means no tone mapping. + float RelativePeak() const; +}; + +/// Describe a frame's color. peak_nits is the content peak from stream +/// metadata (0 = unknown); it is only used for PQ. +SourceColor DescribeFrame(const AVFrame* frame, int width, int height, double peak_nits = 0.0); +/// Content peak from a frame's HDR10 static metadata (MaxCLL, then mastering +/// display maximum), or 0 when absent. +double FramePeakNits(const AVFrame* frame); + +/// CPU conversion of nonlinear R'G'B'A (16-bit, native endian, full range, in +/// the source's primaries and transfer) into SDR BT.709 RGBA8. Lookup tables +/// are built once per SourceColor; Convert is thread-safe. +class SdrConverter { +public: + explicit SdrConverter(const SourceColor& source); + const SourceColor& Source() const { return source; } + void Convert(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const; + /// Convert one linear-light RGB value (1.0 = reference white, source + /// primaries) to SDR display RGB in [0,1]. Exposed for tests. + void ToSdr(float rgb[3]) const; +private: + SourceColor source; + float peak; + std::vector linearize; ///< 12-bit signal -> linear (relative to reference white) + std::vector hlg_gain; ///< HLG OOTF gain by scene luminance + std::vector encode; ///< linear (by float exponent/mantissa) -> BT.1886 8-bit code + std::vector tone_scale; ///< brightest channel -> tone curve scale + template + void ConvertRows(const uint16_t* src, int src_stride_bytes, uint8_t* dst, int dst_stride_bytes, + int width, int height) const; +}; + +/// Tone-map one linear SDR-relative value (max channel) to [0,1]. +float ToneMap(float value, float relative_peak); + +} // namespace color +} // namespace openshot +#endif diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 135e751cb..4c82a003d 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -25,6 +25,11 @@ #include "FFmpegUtilities.h" #include "FFmpegColorRange.h" +#include "ColorManagement.h" + +extern "C" { +#include +} #include "PreviewSize.h" #include "effects/CropHelpers.h" @@ -728,10 +733,26 @@ void FFmpegReader::Open() { } // Process video stream side data (rotation, spherical metadata, etc) + hdr_peak_nits = 0.0; + sdr_converter.reset(); for (unsigned int i = 0; i < pFormatCtx->nb_streams; i++) { AVStream* st = pFormatCtx->streams[i]; if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { size_t side_data_size = 0; + if (static_cast(i) == videoStream) { + // HDR10 static metadata sets the tone mapping peak (MaxCLL, then mastering peak). + const uint8_t *light = ffmpeg_stream_get_side_data( + st, AV_PKT_DATA_CONTENT_LIGHT_LEVEL, &side_data_size); + if (light && side_data_size >= sizeof(AVContentLightMetadata)) + hdr_peak_nits = reinterpret_cast(light)->MaxCLL; + const uint8_t *mastering = ffmpeg_stream_get_side_data( + st, AV_PKT_DATA_MASTERING_DISPLAY_METADATA, &side_data_size); + if (hdr_peak_nits <= 0.0 && mastering && side_data_size >= sizeof(AVMasteringDisplayMetadata)) { + const auto *display = reinterpret_cast(mastering); + if (display->has_luminance && display->max_luminance.den > 0) + hdr_peak_nits = av_q2d(display->max_luminance); + } + } const uint8_t *displaymatrix = ffmpeg_stream_get_side_data( st, AV_PKT_DATA_DISPLAYMATRIX, &side_data_size); if (displaymatrix && @@ -1842,6 +1863,11 @@ bool FFmpegReader::GetAVFrame() { info.vcodec + "; " + path, Logger::LevelInfo); #endif + // HDR10 metadata may only arrive with frames. Keep the first value so + // tone mapping does not change from frame to frame. + if (hdr_peak_nits <= 0.0) + hdr_peak_nits = color::FramePeakNits(decoded_frame); + // Allocate image (align 32 for simd) AVPixelFormat decoded_pix_fmt = (AVPixelFormat)(decoded_frame->format); if (decoded_pix_fmt == AV_PIX_FMT_NONE) @@ -2189,7 +2215,9 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { try { if (!vulkan_processor) vulkan_processor = std::make_shared(); auto f = CreateFrame(current_frame); - f->AddDeferredImage(vulkan_processor->Process(pFrame, width, height, openshot::Settings::Instance()->HIGH_QUALITY_SCALING)); + f->AddDeferredImage(vulkan_processor->Process(pFrame, width, height, + color::DescribeFrame(pFrame, src_width, src_height, hdr_peak_nits), + openshot::Settings::Instance()->HIGH_QUALITY_SCALING)); gpu_metrics::Record(gpu_metrics::RenderGpu); if (ApplyOrientationMetadata() && std::fabs(source_rotation) >= 0.0001) Logger::Instance()->AppendDebugMethod("Vulkan reader: source orientation requires scaled CPU readback"); @@ -2220,21 +2248,36 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { int buffer_size = ((raw_buffer_size + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; buffer = (unsigned char*) aligned_malloc(buffer_size, ALIGNMENT); - // Copy picture data from one AVFrame (or AVPicture) to another one. - AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height); + // HDR (PQ/HLG) and BT.2020 sources keep 16 bits through scaling, then + // convert to SDR BT.709. Other sources convert straight to RGBA8. + const color::SourceColor source_color = color::DescribeFrame(pFrame, src_width, src_height, hdr_peak_nits); + const bool convert_color = source_color.NeedsConversion(); + const AVPixelFormat rgb_pix_fmt = convert_color ? AV_PIX_FMT_RGBA64 : PIX_FMT_RGBA; + if (convert_color) { + hdr_buffer.resize(size_t(width) * height * 4); + av_image_fill_arrays(pFrameRGB->data, pFrameRGB->linesize, + reinterpret_cast(hdr_buffer.data()), AV_PIX_FMT_RGBA64, width, height, 1); + const auto* previous = sdr_converter ? &sdr_converter->Source() : nullptr; + if (!previous || previous->transfer != source_color.transfer || previous->bt2020 != source_color.bt2020 || + previous->RelativePeak() != source_color.RelativePeak()) + sdr_converter = std::make_shared(source_color); + } else { + // Copy picture data from one AVFrame (or AVPicture) to another one. + AV_COPY_PICTURE_DATA(pFrameRGB, buffer, PIX_FMT_RGBA, width, height); + } int scale_mode = SWS_FAST_BILINEAR; if (openshot::Settings::Instance()->HIGH_QUALITY_SCALING) { scale_mode = SWS_BICUBIC; } - img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, PIX_FMT_RGBA, scale_mode, NULL, NULL, NULL); + img_convert_ctx = sws_getCachedContext(img_convert_ctx, src_width, src_height, src_pix_fmt, width, height, rgb_pix_fmt, scale_mode, NULL, NULL, NULL); if (!img_convert_ctx) throw OutOfMemory("Failed to initialize sws context", path); - const int *src_coeff = sws_getCoefficients(SWS_CS_DEFAULT); - const int *dst_coeff = sws_getCoefficients(SWS_CS_DEFAULT); + // Decode with the source's tagged YUV matrix (BT.601 when untagged). + const int *coefficients = sws_getCoefficients(color::SwsColorspace(source_color.matrix)); const int dst_full_range = 1; // RGB outputs are full-range - sws_setColorspaceDetails(img_convert_ctx, src_coeff, src_full_range ? 1 : 0, - dst_coeff, dst_full_range, 0, 1 << 16, 1 << 16); + sws_setColorspaceDetails(img_convert_ctx, coefficients, src_full_range ? 1 : 0, + coefficients, dst_full_range, 0, 1 << 16, 1 << 16); if (!pFrame || !pFrame->data[0] || pFrame->linesize[0] <= 0) { #if USE_HW_ACCEL @@ -2281,6 +2324,9 @@ void FFmpegReader::ProcessVideoPacket(int64_t requested_frame) { return; } + if (convert_color) + sdr_converter->Convert(hdr_buffer.data(), pFrameRGB->linesize[0], buffer, width * bytes_per_pixel, width, height); + // Create or get the existing frame object std::shared_ptr f = CreateFrame(current_frame); diff --git a/src/FFmpegReader.h b/src/FFmpegReader.h index 5c5765697..b7b918e06 100644 --- a/src/FFmpegReader.h +++ b/src/FFmpegReader.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "AudioLocation.h" #include "CacheMemory.h" #include "Clip.h" @@ -103,6 +104,8 @@ namespace openshot { * r.Close(); * @endcode */ + namespace color { class SdrConverter; } + class FFmpegReader : public ReaderBase { private: std::string path; @@ -173,6 +176,9 @@ namespace openshot { // Cached conversion contexts and frames for performance SwsContext *img_convert_ctx = nullptr; ///< Cached video scaler context + double hdr_peak_nits = 0.0; ///< HDR10 content peak from metadata (0 = unknown) + std::shared_ptr sdr_converter; ///< HDR/wide gamut to SDR (CPU path) + std::vector hdr_buffer; ///< 16-bit RGBA scratch for SDR conversion SWRCONTEXT *avr_ctx = nullptr; ///< Cached audio resample context AVFrame *pFrameRGB_cached = nullptr; ///< Temporary frame used for video conversion diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp index 8d236d37b..e0b065b7b 100644 --- a/src/FFmpegWriter.cpp +++ b/src/FFmpegWriter.cpp @@ -28,6 +28,8 @@ #include "FFmpegUtilities.h" #include "FFmpegWriter.h" +#include "FFmpegColorRange.h" +#include "ColorManagement.h" #include "Exceptions.h" #include "Frame.h" #ifdef OPENSHOT_ENABLE_GPU_FRAME @@ -2167,6 +2169,24 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) { } #endif + // Tag YUV outputs so players decode them as rendered: SDR BT.709 color, + // with the matrix players assume for this size (BT.709 for HD, BT.601 for + // SD). RGB outputs are left untagged. + { + const AVPixFmtDescriptor* desc = av_pix_fmt_desc_get(video_codec_ctx->pix_fmt); + const bool hardware = desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL); + const bool rgb = desc && !hardware && (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL)); + const bool gray = desc && !hardware && desc->nb_components <= 2; + if (desc && !rgb && !gray) { + bool full_range = false; + NormalizeDeprecatedPixFmt(video_codec_ctx->pix_fmt, full_range); + video_codec_ctx->color_primaries = AVCOL_PRI_BT709; + video_codec_ctx->color_trc = AVCOL_TRC_BT709; + video_codec_ctx->colorspace = color::EncodeMatrix(info.width, info.height); + video_codec_ctx->color_range = full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + } + } + /* open the codec */ if (avcodec_open2(video_codec_ctx, codec, &opts) < 0) throw InvalidCodec("Could not open video codec", path); @@ -2896,6 +2916,13 @@ void FFmpegWriter::process_video_packet(std::shared_ptr frame) { ); if (!img_convert_ctx) throw ErrorEncodingVideo("Could not initialize sws context", -1); + // Encode with the matrix and range tagged in open_video(). + bool dst_full_range = false; + NormalizeDeprecatedPixFmt(dst_fmt, dst_full_range); + const int* coefficients = sws_getCoefficients( + color::SwsColorspace(color::EncodeMatrix(info.width, info.height))); + sws_setColorspaceDetails(img_convert_ctx, coefficients, 1, coefficients, + dst_full_range ? 1 : 0, 0, 1 << 16, 1 << 16); } // Scale RGBA → dst_fmt into persistent_dst_buffer diff --git a/src/MacVulkanInterop.h b/src/MacVulkanInterop.h index a1a4d53f4..cf0a76cb1 100644 --- a/src/MacVulkanInterop.h +++ b/src/MacVulkanInterop.h @@ -3,6 +3,9 @@ #ifndef OPENSHOT_MAC_VULKAN_INTEROP_H #define OPENSHOT_MAC_VULKAN_INTEROP_H #include +extern "C" { +#include +} struct AVFrame; struct AVBufferRef; namespace openshot { @@ -10,6 +13,9 @@ namespace openshot { // surfaces retain their CVPixelBuffers until all Vulkan submissions have retired. AVFrame* ImportVideoToolboxFrame(const AVFrame* frame, bool writable = false); AVFrame* AllocateMacVulkanFrame(AVBufferRef* device, int width, int height, bool nv12); +/// The FFmpeg layout of a Core Video pixel format (OSType) that can be shared +/// with Vulkan: NV12, P010 or P210, else AV_PIX_FMT_NONE. +AVPixelFormat MacSurfaceFormat(uint32_t core_video_format); uint64_t MacVulkanDeviceRegistryID(const AVFrame* frame); int64_t MacVulkanFrameBytes(const AVFrame* frame); bool IsMacVulkanFrame(const AVFrame* frame); diff --git a/src/MacVulkanInterop.mm b/src/MacVulkanInterop.mm index 5f119e167..92d4ebc3c 100644 --- a/src/MacVulkanInterop.mm +++ b/src/MacVulkanInterop.mm @@ -19,34 +19,59 @@ #include #include #include +#include } namespace openshot { namespace { const char native_pool_tag = 0; void AvCheck(int r, const char* what) { if (r < 0) throw std::runtime_error(what); } void VkCheck(VkResult r, const char* what) { if (r != VK_SUCCESS) throw std::runtime_error(what); } +// Shared surface layouts: RGBA, or two-plane YUV with 8-bit (NV12) or +// MSB-aligned 10-bit (P010 4:2:0, P210 4:2:2) samples, as VideoToolbox decodes. +struct SurfaceFormat { + AVPixelFormat sw_format; + int planes; + MTLPixelFormat metal[2]; + VkFormat vulkan[2]; +}; +SurfaceFormat Describe(AVPixelFormat sw_format) { + switch (sw_format) { + case AV_PIX_FMT_NV12: + return {sw_format, 2, {MTLPixelFormatR8Unorm, MTLPixelFormatRG8Unorm}, {VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM}}; + case AV_PIX_FMT_P010: + case AV_PIX_FMT_P210: + return {sw_format, 2, {MTLPixelFormatR16Unorm, MTLPixelFormatRG16Unorm}, {VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM}}; + case AV_PIX_FMT_RGBA: + return {sw_format, 1, {MTLPixelFormatRGBA8Unorm, MTLPixelFormatInvalid}, {VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_UNDEFINED}}; + default: + throw std::runtime_error("Unsupported Mac Vulkan surface format"); + } +} struct Pools { std::mutex mutex; - std::map, AVBufferRef*> frames; + std::map, AVBufferRef*> frames; ~Pools() { for (auto& pair : frames) av_buffer_unref(&pair.second); } }; Pools& NativePools() { static Pools pools; return pools; } -AVBufferRef* Pool(AVBufferRef* device, int width, int height, bool nv12) { +AVBufferRef* Pool(AVBufferRef* device, int width, int height, AVPixelFormat sw_format) { auto& pools = NativePools(); std::lock_guard lock(pools.mutex); - auto key = std::make_tuple(reinterpret_cast(device->data), width, height, nv12); + auto key = std::make_tuple(reinterpret_cast(device->data), width, height, sw_format); auto& ref = pools.frames[key]; if (!ref) { ref = av_hwframe_ctx_alloc(device); if (!ref) throw std::bad_alloc(); auto* fc = reinterpret_cast(ref->data); fc->format = AV_PIX_FMT_VULKAN; - fc->sw_format = nv12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_RGBA; + fc->sw_format = sw_format; fc->width = width; fc->height = height; fc->user_opaque = const_cast(&native_pool_tag); auto* vc = static_cast(fc->hwctx); vc->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE; - vc->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | + // These allocations only carry FFmpeg's synchronization; Wrap substitutes + // the imported images. 10-bit surfaces are only read (decoded video). + const bool wide = sw_format == AV_PIX_FMT_P010 || sw_format == AV_PIX_FMT_P210; + vc->usage = static_cast(VK_IMAGE_USAGE_SAMPLED_BIT | (wide ? 0 : VK_IMAGE_USAGE_STORAGE_BIT) | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); const int result = av_hwframe_ctx_init(ref); if (result < 0) { av_buffer_unref(&ref); pools.frames.erase(key); AvCheck(result, "Initialize Mac Vulkan frame pool"); } @@ -108,7 +133,9 @@ void Wait() const { if (fc->user_opaque != &native_pool_tag) return nullptr; return static_cast(av_buffer_get_opaque(frame->buf[0])); } -AVFrame* Wrap(AVBufferRef* device, CVPixelBufferRef pixel, bool nv12, bool writable) { +AVFrame* Wrap(AVBufferRef* device, CVPixelBufferRef pixel, AVPixelFormat sw_format, bool writable) { + const SurfaceFormat surface = Describe(sw_format); + const bool planar = surface.planes == 2; @autoreleasepool { const int width = static_cast(CVPixelBufferGetWidth(pixel)); const int height = static_cast(CVPixelBufferGetHeight(pixel)); @@ -116,10 +143,10 @@ void Wait() const { throw std::invalid_argument("Mac GPU frames require IOSurface-backed pixels"); auto state = std::make_unique(); state->pixel = CVPixelBufferRetain(pixel); - state->count = nv12 ? 2 : 1; + state->count = surface.planes; state->allocation = av_frame_alloc(); if (!state->allocation) throw std::bad_alloc(); - AVBufferRef* pool = Pool(device, width, height, nv12); + AVBufferRef* pool = Pool(device, width, height, sw_format); const int allocated = av_hwframe_get_buffer(pool, state->allocation, 0); av_buffer_unref(&pool); AvCheck(allocated, "Allocate Mac Vulkan frame synchronization"); @@ -141,10 +168,10 @@ void Wait() const { NSDictionary* attributes = @{(id)kCVMetalTextureUsage: @(MTLTextureUsageShaderRead | (writable ? MTLTextureUsageShaderWrite : 0))}; for (int i = 0; i < state->count; ++i) { - const auto w = nv12 ? CVPixelBufferGetWidthOfPlane(pixel, i) : size_t(width); - const auto h = nv12 ? CVPixelBufferGetHeightOfPlane(pixel, i) : size_t(height); - const auto format = nv12 ? (i ? MTLPixelFormatRG8Unorm : MTLPixelFormatR8Unorm) : MTLPixelFormatRGBA8Unorm; - if (nv12) { + const auto w = planar ? CVPixelBufferGetWidthOfPlane(pixel, i) : size_t(width); + const auto h = planar ? CVPixelBufferGetHeightOfPlane(pixel, i) : size_t(height); + const auto format = surface.metal[i]; + if (planar) { if (CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, state->cache, pixel, (CFDictionaryRef)attributes, format, w, h, i, &state->textures[i]) != kCVReturnSuccess) throw std::runtime_error("Map Core Video pixel planes to Metal"); @@ -164,7 +191,7 @@ void Wait() const { texture.mtlTexture = state->Texture(i); VkImageCreateInfo create{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; create.pNext = &texture; create.imageType = VK_IMAGE_TYPE_2D; - create.format = nv12 ? (i ? VK_FORMAT_R8G8_UNORM : VK_FORMAT_R8_UNORM) : VK_FORMAT_R8G8B8A8_UNORM; + create.format = surface.vulkan[i]; create.extent = {uint32_t(w), uint32_t(h), 1}; create.mipLevels = create.arrayLayers = 1; create.samples = VK_SAMPLE_COUNT_1_BIT; create.tiling = VK_IMAGE_TILING_OPTIMAL; create.usage = VK_IMAGE_USAGE_SAMPLED_BIT | (writable ? VK_IMAGE_USAGE_STORAGE_BIT : 0); @@ -174,7 +201,7 @@ void Wait() const { for (int i = 0; i < state->count; ++i) { state->image->img[i] = state->imported[i]; state->image->mem[i] = VK_NULL_HANDLE; - state->image->size[i] = nv12 ? CVPixelBufferGetBytesPerRowOfPlane(pixel, i) * CVPixelBufferGetHeightOfPlane(pixel, i) + state->image->size[i] = planar ? CVPixelBufferGetBytesPerRowOfPlane(pixel, i) * CVPixelBufferGetHeightOfPlane(pixel, i) : CVPixelBufferGetBytesPerRow(pixel) * height; state->image->offset[i] = 0; state->image->layout[i] = writable ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_PREINITIALIZED; @@ -215,6 +242,22 @@ int64_t MacVulkanFrameBytes(const AVFrame* frame) { return static_cast(std::min(size, std::numeric_limits::max())); } bool IsMacVulkanFrame(const AVFrame* frame) { return State(frame) != nullptr; } +AVPixelFormat MacSurfaceFormat(uint32_t core_video_format) { + switch (core_video_format) { + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + return AV_PIX_FMT_NV12; + // 10-bit HEVC/AV1 decode (iPhone HDR): 16-bit samples, data in the high bits. + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + return AV_PIX_FMT_P010; + case kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: + return AV_PIX_FMT_P210; + default: + return AV_PIX_FMT_NONE; + } +} void WaitMacVulkanFrame(const AVFrame* frame) { auto* state = State(frame); if (!state) throw std::invalid_argument("Not a shared Mac Vulkan frame"); @@ -225,13 +268,13 @@ void WaitMacVulkanFrame(const AVFrame* frame) { if (!frame || frame->format != AV_PIX_FMT_VIDEOTOOLBOX || !frame->data[3]) throw std::invalid_argument("Expected a VideoToolbox frame"); auto pixel = reinterpret_cast(frame->data[3]); - const auto type = CVPixelBufferGetPixelFormatType(pixel); - if (type != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && type != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) - throw std::runtime_error("Mac Vulkan currently supports 8-bit NV12 decode/encode surfaces"); + const AVPixelFormat sw_format = MacSurfaceFormat(CVPixelBufferGetPixelFormatType(pixel)); + if (sw_format == AV_PIX_FMT_NONE) + throw std::runtime_error("Mac Vulkan supports NV12, P010 and P210 decode/encode surfaces"); AVBufferRef* device = nullptr; AvCheck(CreateSharedVulkanDevice(&device), "Create Mac compositor device"); AVFrame* result = nullptr; - try { result = Wrap(device, pixel, true, writable); } catch (...) { av_buffer_unref(&device); throw; } + try { result = Wrap(device, pixel, sw_format, writable); } catch (...) { av_buffer_unref(&device); throw; } av_buffer_unref(&device); const int props = av_frame_copy_props(result, frame); if (props < 0) { av_frame_free(&result); AvCheck(props, "Copy VideoToolbox metadata"); } @@ -245,7 +288,7 @@ void WaitMacVulkanFrame(const AVFrame* frame) { if (CVPixelBufferCreate(kCFAllocatorDefault, width, height, type, (CFDictionaryRef)attrs, &pixel) != kCVReturnSuccess) throw std::runtime_error("Allocate shared compositor IOSurface"); AVFrame* frame = nullptr; - try { frame = Wrap(device, pixel, nv12, true); } catch (...) { CVPixelBufferRelease(pixel); throw; } + try { frame = Wrap(device, pixel, nv12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_RGBA, true); } catch (...) { CVPixelBufferRelease(pixel); throw; } CVPixelBufferRelease(pixel); return frame; } @@ -276,11 +319,11 @@ int ReadMacVulkanFrame(AVFrame* destination, const AVFrame* source) { if (!completed) return AVERROR_EXTERNAL; } if (CVPixelBufferLockBaseAddress(state->pixel, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) return AVERROR_EXTERNAL; - const bool nv12 = destination->format == AV_PIX_FMT_NV12; + const bool planar = state->count == 2; const uint8_t* data[4]{}; int strides[4]{}; - for (int i = 0; i < (nv12 ? 2 : 1); ++i) { - data[i] = static_cast(nv12 ? CVPixelBufferGetBaseAddressOfPlane(state->pixel, i) : CVPixelBufferGetBaseAddress(state->pixel)); - strides[i] = static_cast(nv12 ? CVPixelBufferGetBytesPerRowOfPlane(state->pixel, i) : CVPixelBufferGetBytesPerRow(state->pixel)); + for (int i = 0; i < state->count; ++i) { + data[i] = static_cast(planar ? CVPixelBufferGetBaseAddressOfPlane(state->pixel, i) : CVPixelBufferGetBaseAddress(state->pixel)); + strides[i] = static_cast(planar ? CVPixelBufferGetBytesPerRowOfPlane(state->pixel, i) : CVPixelBufferGetBytesPerRow(state->pixel)); } av_image_copy(destination->data, destination->linesize, data, strides, AVPixelFormat(destination->format), source->width, source->height); CVPixelBufferUnlockBaseAddress(state->pixel, kCVPixelBufferLock_ReadOnly); diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 81cfcbae0..81dd7941c 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -4,6 +4,7 @@ #include "VulkanEncodeConverter.h" #include "GpuDevices.h" #include "VulkanFrameImage.h" +#include "ColorManagement.h" #ifdef __APPLE__ #include "MacVulkanInterop.h" #endif @@ -22,6 +23,7 @@ extern "C" { #include #include #include +#include } // SPIR-V generated from src/vulkan/*.comp at build time. #include "affine.comp.h" @@ -301,7 +303,7 @@ class Pipeline { } std::shared_ptr Render(const VulkanFrameImage& input,const VulkanFrameImage* background, int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality=false,const AVFrame* target=nullptr, - const EffectParameters* effect=nullptr) { + const EffectParameters* effect=nullptr,const color::SourceColor* source_color=nullptr) { const auto* src=input.GetFrame(); const auto* src_fc=reinterpret_cast(src->hw_frames_ctx->data); if(src_fc->device_ctx!=device_context) throw std::runtime_error("Vulkan source device mismatch"); @@ -353,7 +355,10 @@ class Pipeline { #endif } AvCheck(av_frame_copy_props(output.get(),src),"Copy compositor metadata"); - output->colorspace=encode_nv12?AVCOL_SPC_SMPTE170M:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; + // Export NV12 uses the matrix FFmpegWriter tags for this size (BT.709 for HD). + const AVColorSpace encode_matrix=color::EncodeMatrix(width,height); + output->colorspace=encode_nv12?encode_matrix:AVCOL_SPC_RGB; output->color_range=encode_nv12?AVCOL_RANGE_MPEG:AVCOL_RANGE_JPEG; + if(encode_nv12){output->color_primaries=AVCOL_PRI_BT709;output->color_trc=AVCOL_TRC_BT709;} if(encode_nv12 && src_fc->sw_format!=AV_PIX_FMT_RGBA) throw std::runtime_error("Encoder conversion requires RGBA"); Parameters p{}; bool invertible=false; @@ -375,19 +380,45 @@ class Pipeline { p.matrix_x[3]=effect->values[8]; } int plane_count=0; - if(src_fc->sw_format==AV_PIX_FMT_RGBA){p.modes[0]=input.IsPremultiplied()?2:3;plane_count=1;} - else if(src_fc->sw_format==AV_PIX_FMT_NV12){p.modes[0]=1;plane_count=1;} - else if(src_fc->sw_format==AV_PIX_FMT_YUV420P){p.modes[0]=0;plane_count=3;} - else throw std::runtime_error("Vulkan affine renderer supports RGBA, NV12 and YUV420P"); + const AVPixelFormat sw_format=src_fc->sw_format; + // Two-plane 4:2:0/4:2:2 with MSB-aligned 10-16 bit samples (HEVC/AV1 Main10 decode). + const bool wide=sw_format==AV_PIX_FMT_P010 || sw_format==AV_PIX_FMT_P012 || sw_format==AV_PIX_FMT_P016 || + sw_format==AV_PIX_FMT_P210 || sw_format==AV_PIX_FMT_P212 || sw_format==AV_PIX_FMT_P216; + if(sw_format==AV_PIX_FMT_RGBA){p.modes[0]=input.IsPremultiplied()?2:3;plane_count=1;} + else if(sw_format==AV_PIX_FMT_NV12 || wide){p.modes[0]=1;plane_count=1;} + else if(sw_format==AV_PIX_FMT_YUV420P){p.modes[0]=0;plane_count=3;} + else throw std::runtime_error(std::string("Vulkan affine renderer does not support ")+ + (av_get_pix_fmt_name(sw_format)?av_get_pix_fmt_name(sw_format):"this format")); + const AVColorSpace matrix_space=source_color?source_color->matrix:color::DecodeMatrix(src->colorspace,src->width,src->height); double kr=0.299,kb=0.114; - if(src->colorspace==AVCOL_SPC_BT709){kr=0.2126;kb=0.0722;} - else if(src->colorspace==AVCOL_SPC_BT2020_NCL){kr=0.2627;kb=0.0593;} - else if(src->colorspace==AVCOL_SPC_SMPTE240M){kr=0.212;kb=0.087;} - const bool full=src->color_range==AVCOL_RANGE_JPEG; - const double cscale=full?1.0:255.0/224.0; - p.coeffs[0]=full?1.0:255.0/219.0;p.extra[1]=full?0.0:16.0/255.0;p.extra[2]=128.0/255.0; + if(matrix_space==AVCOL_SPC_BT709){kr=0.2126;kb=0.0722;} + else if(matrix_space==AVCOL_SPC_BT2020_NCL || matrix_space==AVCOL_SPC_BT2020_CL){kr=0.2627;kb=0.0593;} + else if(matrix_space==AVCOL_SPC_SMPTE240M){kr=0.212;kb=0.087;} + else if(matrix_space==AVCOL_SPC_FCC){kr=0.30;kb=0.11;} + const bool full=source_color?source_color->full_range:src->color_range==AVCOL_RANGE_JPEG; + // Samples are normalized by the texture's maximum (255 or 65535). Wide + // formats hold codes in the high bits, so limited range scales by 256. + const int depth=wide?av_pix_fmt_desc_get(sw_format)->comp[0].depth:8; + const double unit=wide?65535.0:255.0; + const double maximum=wide?double((1<NeedsConversion() && p.modes[0]<2) { + color_flags=(int(source_color->transfer)<<4)|(source_color->bt2020?256:0); + p.matrix_y[3]=source_color->RelativePeak(); + } // Retire before locking inputs: releasing the last imported surface can // wait on that frame's synchronization mutex. Slot& slot=ring[next_slot]; @@ -440,8 +471,8 @@ class Pipeline { auto view=View(image,format,aspect);owned_views.push_back(view);return view; }; if(p.modes[0]==1){ - views[0]=make_view(sf->img[0],VK_FORMAT_R8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); - views[1]=views[2]=make_view(sf->img[separate_nv12?1:0],VK_FORMAT_R8G8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); + views[0]=make_view(sf->img[0],wide?VK_FORMAT_R16_UNORM:VK_FORMAT_R8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); + views[1]=views[2]=make_view(sf->img[separate_nv12?1:0],wide?VK_FORMAT_R16G16_UNORM:VK_FORMAT_R8G8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_1_BIT); }else if(p.modes[0]>=2){ views[0]=views[1]=views[2]=make_view(sf->img[0],VK_FORMAT_R8G8B8A8_UNORM,VK_IMAGE_ASPECT_COLOR_BIT); }else{ @@ -511,6 +542,7 @@ class Pipeline { vkCmdPipelineBarrier(cmd,VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,0,0,nullptr,0,nullptr,barriers.size(),barriers.data()); vkCmdBindPipeline(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,pipeline); vkCmdBindDescriptorSets(cmd,VK_PIPELINE_BIND_POINT_COMPUTE,layout,0,1,&set,0,nullptr); + p.modes[0]|=color_flags; vkCmdPushConstants(cmd,layout,VK_SHADER_STAGE_COMPUTE_BIT,0,sizeof(p),&p); vkCmdDispatch(cmd,(width+15)/16,(height+15)/16,1); if(scope) { @@ -561,7 +593,7 @@ class Pipeline { } static std::shared_ptr RenderImage(std::shared_ptr source, std::shared_ptr background,int width,int height,const QTransform& matrix,double opacity,const QColor& clear,bool high_quality, - const EffectParameters* effect){ + const EffectParameters* effect,const color::SourceColor* source_color=nullptr){ auto src=std::dynamic_pointer_cast(source); auto back=std::dynamic_pointer_cast(background); if(!src || (background && !back)) throw std::invalid_argument("Vulkan compositor requires Vulkan surfaces"); @@ -584,7 +616,13 @@ static std::shared_ptr RenderImage(std::shared_ptrsecond.used=++clock; auto& pipeline=found->second.pipeline; - return pipeline->Render(*src,back.get(),width,height,matrix,opacity,clear,high_quality,nullptr,effect); + return pipeline->Render(*src,back.get(),width,height,matrix,opacity,clear,high_quality,nullptr,effect,source_color); +} +std::shared_ptr DecodeVulkanImage(std::shared_ptr source, + int width,int height,const color::SourceColor& source_color,bool high_quality) { + QTransform matrix; + matrix.scale(double(width)/source->GetWidth(),double(height)/source->GetHeight()); + return RenderImage(source,nullptr,width,height,matrix,1.0,Qt::transparent,high_quality,nullptr,&source_color); } std::shared_ptr RenderVulkanImage(std::shared_ptr source, std::shared_ptr background,int width,int height,const QTransform& matrix, diff --git a/src/VulkanCompositor.h b/src/VulkanCompositor.h index f8bfac744..23eeee44c 100644 --- a/src/VulkanCompositor.h +++ b/src/VulkanCompositor.h @@ -11,6 +11,12 @@ #include #include namespace openshot { +namespace color { struct SourceColor; } +/// Scale a decoded YUV surface to RGBA using its color description: YUV +/// matrix and range, and for HDR or BT.2020 sources conversion to SDR BT.709 +/// matching color::SdrConverter on the CPU. +std::shared_ptr DecodeVulkanImage(std::shared_ptr source, + int width, int height, const color::SourceColor& source_color, bool high_quality = false); /// Affine transform and source-over compositing into an independently owned GPU frame. /// Inputs must be Vulkan surfaces from the same device; null background uses clear. /// high_quality selects antialiased bicubic reader resampling; default matches Qt bilinear transforms. diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index 4fe4d17eb..f2b646458 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -170,7 +170,8 @@ struct VulkanFrameProcessor::Impl {}; VulkanFrameProcessor::VulkanFrameProcessor() : impl(new Impl) {} VulkanFrameProcessor::~VulkanFrameProcessor() = default; void VulkanFrameProcessor::Reset() {} -std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* frame, int width, int height, bool high_quality) { +std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* frame, int width, int height, + const color::SourceColor& color, bool high_quality) { #ifdef __APPLE__ OwnedFrame imported; if (frame && frame->format == AV_PIX_FMT_VIDEOTOOLBOX) { @@ -180,15 +181,8 @@ std::shared_ptr VulkanFrameProcessor::Process(const AVFrame* f #endif Validate(frame); if (width <= 0 || height <= 0) throw std::invalid_argument("Invalid Vulkan output dimensions"); - // Preserve the existing FFmpegReader conversion matrix (SWS_CS_DEFAULT / BT.601). - // The generic compositor still honors metadata for callers outside that reader. - auto compatible = AllocateFrame(); - Check(av_frame_ref(compatible.get(), frame), "Reference reader Vulkan frame"); - compatible->colorspace = AVCOL_SPC_SMPTE170M; - auto source = std::make_shared(compatible.get()); - QTransform matrix; - matrix.scale(double(width) / frame->width, double(height) / frame->height); - return RenderVulkanImage(source, nullptr, width, height, matrix, 1.0, Qt::transparent, high_quality); + auto source = std::make_shared(frame); + return DecodeVulkanImage(source, width, height, color, high_quality); } namespace { struct BufferDeleter { void operator()(AVBufferRef* p) const { av_buffer_unref(&p); } }; diff --git a/src/VulkanFrameImage.h b/src/VulkanFrameImage.h index 2ef17590b..81ca2753e 100644 --- a/src/VulkanFrameImage.h +++ b/src/VulkanFrameImage.h @@ -7,6 +7,7 @@ #include struct AVFrame; namespace openshot { +namespace color { struct SourceColor; } /// False when OPENSHOT_RENDER_BACKEND=cpu selects the CPU pipeline at runtime. bool VulkanRenderingEnabled(); /// Process-wide count of GPU images materialized as CPU pixels (diagnostics). @@ -45,7 +46,10 @@ class VulkanFrameProcessor { ~VulkanFrameProcessor(); VulkanFrameProcessor(const VulkanFrameProcessor&) = delete; VulkanFrameProcessor& operator=(const VulkanFrameProcessor&) = delete; - std::shared_ptr Process(const AVFrame* frame, int width, int height, bool high_quality = false); + /// Scale and convert a decoded frame to RGBA. color describes the frame + /// (see color::DescribeFrame); HDR and BT.2020 sources become SDR BT.709. + std::shared_ptr Process(const AVFrame* frame, int width, int height, + const color::SourceColor& color, bool high_quality = false); void Reset(); private: struct Impl; diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp index 75e4de826..9b8a9ce5d 100644 --- a/src/vulkan/affine.comp +++ b/src/vulkan/affine.comp @@ -16,7 +16,8 @@ layout(push_constant) uniform Parameters { vec4 extra; // U->B, Y offset, chroma offset, opacity vec4 clear; ivec4 sizes; // output width/height, source width/height - ivec4 modes; // 0 planar, 1 NV12, 2 premult RGBA, 3 straight RGBA; background enabled + ivec4 modes; // x: source format (bits 0-3: 0 planar, 1 NV12/P01x, 2 premult RGBA, 3 straight RGBA) + // plus transfer (bits 4-7) and BT.2020 gamut (bit 8), see to_sdr(); y: background enabled vec4 effect; // saturation controls, or crop bounds in output pixels } p; // Mitchell/Netravali family B=0, C=0.6 matches libswscale's default @@ -59,20 +60,64 @@ float grade_curve(int curve,float value) { int index=clamp(int(value*255.0+0.5),0,255); return uintBitsToFloat(data[21+curve*256+index]); } +// HDR/wide gamut to SDR BT.709; mirrors color::SdrConverter (ColorManagement.cpp). +// Linear values are relative to the BT.2408 reference white (203 nits). +#define TONE_KNEE 0.8 +#define REFERENCE_WHITE 203.0 +const mat3 BT2020_TO_709=mat3( // column-major + 1.6604910,-0.1245505,-0.0181508, + -0.5876411, 1.1328999,-0.1005789, + -0.0728499,-0.0083494, 1.1187297); +vec3 pq_to_linear(vec3 e) { + const float m1=2610.0/16384.0,m2=2523.0/4096.0*128.0; + const float c1=3424.0/4096.0,c2=2413.0/4096.0*32.0,c3=2392.0/4096.0*32.0; + vec3 v=pow(clamp(e,0.0,1.0),vec3(1.0/m2)); + return 10000.0/REFERENCE_WHITE*pow(max(v-c1,0.0)/(c2-c3*v),vec3(1.0/m1)); +} +vec3 hlg_to_scene(vec3 e) { + const float a=0.17883277,b=0.28466892,c=0.55991073; + e=clamp(e,0.0,1.0); + return mix(e*e/3.0,(exp((e-c)/a)+b)/12.0,greaterThan(e,vec3(0.5))); +} +float tone_map(float value,float peak) { + if(value<=TONE_KNEE || peak<=1.0) return min(value,1.0); + float span=1.0-TONE_KNEE,t=(value-TONE_KNEE)/span; + float w=max((peak-TONE_KNEE)/span,1.0); + return min(TONE_KNEE+span*t*(1.0+t/(w*w))/(1.0+t),1.0); +} +vec3 to_sdr(vec3 signal) { + int transfer=(p.modes.x>>4)&15; + bool bt2020=(p.modes.x&256)!=0; + vec3 rgb; + if(transfer==1) rgb=pq_to_linear(signal); + else if(transfer==2) { + // HLG OOTF for the nominal 1000 nit display (system gamma 1.2). + vec3 scene=hlg_to_scene(signal); + float luma=dot(scene,bt2020?vec3(0.2627,0.6780,0.0593):vec3(0.2126,0.7152,0.0722)); + rgb=scene*(1000.0/REFERENCE_WHITE)*pow(clamp(luma,0.0,1.0),0.2); + } else rgb=pow(clamp(signal,0.0,1.0),vec3(2.4)); + if(bt2020) rgb=BT2020_TO_709*rgb; + rgb=max(rgb,0.0); + float brightest=max(rgb.r,max(rgb.g,rgb.b)); + if(brightest>TONE_KNEE) rgb*=tone_map(brightest,p.matrix_y.w)/brightest; + return pow(min(rgb,1.0),vec3(1.0/2.4)); +} // Premultiplied RGBA from any supported source format. vec4 load_source(vec2 uv) { - if(p.modes.x>=2) { + if((p.modes.x&15)>=2) { vec4 front=clamp(sample_source(plane_y,uv),0.0,1.0); - if(p.modes.x==3) front.rgb*=front.a; + if((p.modes.x&15)==3) front.rgb*=front.a; else front.rgb=min(front.rgb,vec3(front.a)); return front; } float y=(sample_source(plane_y,uv).r-p.extra.y)*p.color_coeffs.x; - vec2 chroma=p.modes.x==1 ? sample_source(plane_u,uv).rg : vec2(sample_source(plane_u,uv).r,sample_source(plane_v,uv).r); + vec2 chroma=(p.modes.x&15)==1 ? sample_source(plane_u,uv).rg : vec2(sample_source(plane_u,uv).r,sample_source(plane_v,uv).r); chroma-=p.extra.z; - return vec4(clamp(vec3(y+p.color_coeffs.y*chroma.y, + vec3 rgb=clamp(vec3(y+p.color_coeffs.y*chroma.y, y+p.color_coeffs.z*chroma.x+p.color_coeffs.w*chroma.y, - y+p.extra.x*chroma.x),0,1),1); + y+p.extra.x*chroma.x),0,1); + if((p.modes.x>>4)!=0) rgb=to_sdr(rgb); + return vec4(rgb,1); } // ColorMap table: domain min/max, then RGB entries. vec3 lut_entry(int index) { int i=6+index*3; return vec3(value(i),value(i+1),value(i+2)); } diff --git a/src/vulkan/encode_nv12.comp b/src/vulkan/encode_nv12.comp index 02da26015..91375fa2d 100644 --- a/src/vulkan/encode_nv12.comp +++ b/src/vulkan/encode_nv12.comp @@ -7,11 +7,14 @@ layout(push_constant) uniform Parameters { vec4 matrix_x, matrix_y, coeffs, extra, clear; ivec4 sizes, modes; } p; -// Match the writer's existing SWS_CS_DEFAULT (BT.601), limited-range output. +// Limited-range output with the writer's matrix (coeffs.xy = Kr, Kb), as +// FFmpegWriter's swscale path encodes. vec3 yuv(vec3 rgb) { - return vec3(dot(rgb,vec3(0.299,0.587,0.114))*219.0/255.0+16.0/255.0, - dot(rgb,vec3(-0.168736,-0.331264,0.5))*224.0/255.0+128.0/255.0, - dot(rgb,vec3(0.5,-0.418688,-0.081312))*224.0/255.0+128.0/255.0); + float kr=p.coeffs.x,kb=p.coeffs.y; + float y=dot(rgb,vec3(kr,1.0-kr-kb,kb)); + return vec3(y*219.0/255.0+16.0/255.0, + (rgb.b-y)/(2.0*(1.0-kb))*224.0/255.0+128.0/255.0, + (rgb.r-y)/(2.0*(1.0-kr))*224.0/255.0+128.0/255.0); } void main() { ivec2 at=ivec2(gl_GlobalInvocationID.xy); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 276a324e1..c909e5f55 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -48,6 +48,7 @@ set(OPENSHOT_TESTS Caption Clip Color + ColorManagement Coordinate DummyReader FFmpegCapabilities @@ -194,6 +195,10 @@ foreach(tname ${OPENSHOT_TESTS}) if(tname MATCHES "^Vulkan") # Same Vulkan headers as the library (newer than some system packages) target_include_directories(openshot-${tname}-test PRIVATE ${Vulkan_INCLUDE_DIR}) + if(APPLE) + # Tests build Core Video surfaces to exercise VideoToolbox interop. + target_link_libraries(openshot-${tname}-test "-framework CoreVideo" "-framework CoreFoundation") + endif() # Report missing GPU, display or interop support as skipped, not passed. list(APPEND test_properties SKIP_REGULAR_EXPRESSION "Skipping Vulkan test:" diff --git a/tests/ColorManagement.cpp b/tests/ColorManagement.cpp new file mode 100644 index 000000000..e03c15b1c --- /dev/null +++ b/tests/ColorManagement.cpp @@ -0,0 +1,222 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include +#include +#include +#include +#include +#include +#include + +#include "openshot_catch.h" + +#include "ColorManagement.h" +#include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include "Frame.h" +#include "Settings.h" + +extern "C" { +#include +#include +#include +} + +using namespace openshot; + +namespace { + +// Patches of examples/color-chart-*.mp4 (see tests/generate-color-charts.py): +// SDR code values, then 600 and 1000 nit highlights. 4x4 grid of 80x48 patches. +struct Patch { const char* name; std::array code; double nits; }; +const Patch kPatches[] = { + {"reference white", {255, 255, 255}, 0}, {"18% grey", {118, 118, 118}, 0}, + {"light skin", {224, 172, 140}, 0}, {"dark skin", {141, 85, 62}, 0}, + {"sky blue", {98, 155, 210}, 0}, {"foliage", {87, 122, 59}, 0}, + {"red", {220, 30, 30}, 0}, {"green", {40, 200, 40}, 0}, {"blue", {30, 40, 220}, 0}, + {"yellow", {240, 220, 40}, 0}, {"near black", {20, 20, 20}, 0}, + {"600 nit highlight", {}, 600}, {"1000 nit highlight", {}, 1000}, +}; + +// The SDR value OpenShot should produce: BT.2408 placement inverted, with +// the shared tone curve for HDR charts (relative_peak > 1). +std::array Expected(const Patch& patch, float relative_peak) { + float rgb[3]; + for (int c = 0; c < 3; ++c) + rgb[c] = patch.nits > 0 ? float(patch.nits / color::kReferenceWhiteNits) + : float(std::pow(patch.code[c] / 255.0, 2.4)); + const float brightest = std::max(rgb[0], std::max(rgb[1], rgb[2])); + const float scale = relative_peak > 1.0f ? color::ToneMap(brightest, relative_peak) / brightest + : std::min(brightest, 1.0f) / brightest; + std::array result; + for (int c = 0; c < 3; ++c) + result[c] = int(std::lround(255.0 * std::pow(std::min(rgb[c] * scale, 1.0f), 1.0 / 2.4))); + return result; +} + +std::array PatchColor(std::shared_ptr frame, int index) { + const int x0 = (index % 4) * 80 + 30, y0 = (index / 4) * 48 + 18; + long sum[3] = {0, 0, 0}; + for (int y = y0; y < y0 + 12; ++y) { + const unsigned char* row = frame->GetPixels(y); + for (int x = x0; x < x0 + 20; ++x) + for (int c = 0; c < 3; ++c) sum[c] += row[x * 4 + c]; + } + return {int(sum[0] / 240), int(sum[1] / 240), int(sum[2] / 240)}; +} + +void CheckChart(const std::string& file, float relative_peak, int tolerance) { + FFmpegReader reader(TEST_MEDIA_PATH + file); + reader.Open(); + auto frame = reader.GetFrame(2); + REQUIRE(frame->GetWidth() == 320); + REQUIRE(frame->GetHeight() == 192); + for (int i = 0; i < int(std::size(kPatches)); ++i) { + const auto expected = Expected(kPatches[i], relative_peak); + const auto actual = PatchColor(frame, i); + INFO(file << " " << kPatches[i].name << " expected (" << expected[0] << "," << expected[1] << "," + << expected[2] << ") got (" << actual[0] << "," << actual[1] << "," << actual[2] << ")"); + for (int c = 0; c < 3; ++c) CHECK(std::abs(actual[c] - expected[c]) <= tolerance); + } + reader.Close(); +} + +// Decode and convert on the CPU. Vulkan builds ignore HARDWARE_DECODER and +// select the CPU pipeline with OPENSHOT_RENDER_BACKEND=cpu. +struct SoftwareDecode { + int previous = Settings::Instance()->HARDWARE_DECODER; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + std::string backend; + bool had_backend = false; +#endif + SoftwareDecode() { + Settings::Instance()->HARDWARE_DECODER = 0; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (const char* value = std::getenv("OPENSHOT_RENDER_BACKEND")) { backend = value; had_backend = true; } + setenv("OPENSHOT_RENDER_BACKEND", "cpu", 1); +#endif + } + ~SoftwareDecode() { + Settings::Instance()->HARDWARE_DECODER = previous; +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (had_backend) setenv("OPENSHOT_RENDER_BACKEND", backend.c_str(), 1); + else unsetenv("OPENSHOT_RENDER_BACKEND"); +#endif + } +}; + +} // namespace + +TEST_CASE("YUV matrices for decoding and encoding", "[libopenshot][color]") { + CHECK(color::DecodeMatrix(AVCOL_SPC_BT709, 640, 480) == AVCOL_SPC_BT709); + CHECK(color::DecodeMatrix(AVCOL_SPC_BT2020_NCL, 3840, 2160) == AVCOL_SPC_BT2020_NCL); + // Untagged files keep BT.601, as earlier OpenShot exports were written. + CHECK(color::DecodeMatrix(AVCOL_SPC_UNSPECIFIED, 1920, 1080) == AVCOL_SPC_SMPTE170M); + CHECK(color::EncodeMatrix(1920, 1080) == AVCOL_SPC_BT709); + CHECK(color::EncodeMatrix(1280, 720) == AVCOL_SPC_BT709); + CHECK(color::EncodeMatrix(720, 576) == AVCOL_SPC_SMPTE170M); + CHECK(color::EncodeMatrix(640, 480) == AVCOL_SPC_SMPTE170M); +} + +TEST_CASE("Frames describe HDR transfer and gamut", "[libopenshot][color]") { + std::unique_ptr frame(av_frame_alloc(), [](AVFrame* f) { av_frame_free(&f); }); + REQUIRE(frame); + CHECK_FALSE(color::DescribeFrame(frame.get(), 1920, 1080).NeedsConversion()); + + frame->color_trc = AVCOL_TRC_ARIB_STD_B67; + frame->color_primaries = AVCOL_PRI_BT2020; + frame->colorspace = AVCOL_SPC_BT2020_NCL; + auto hlg = color::DescribeFrame(frame.get(), 1920, 1080); + CHECK(hlg.transfer == color::Transfer::HLG); + CHECK(hlg.bt2020); + CHECK(hlg.matrix == AVCOL_SPC_BT2020_NCL); + CHECK(hlg.RelativePeak() == Approx(1000.0 / 203.0)); + + frame->color_trc = AVCOL_TRC_SMPTE2084; + auto pq = color::DescribeFrame(frame.get(), 1920, 1080, 4000.0); + CHECK(pq.transfer == color::Transfer::PQ); + CHECK(pq.RelativePeak() == Approx(4000.0 / 203.0)); + + // BT.2020 SDR still needs gamut conversion; unspecified primaries follow the matrix. + frame->color_trc = AVCOL_TRC_BT2020_10; + frame->color_primaries = AVCOL_PRI_UNSPECIFIED; + auto wide = color::DescribeFrame(frame.get(), 1920, 1080); + CHECK(wide.transfer == color::Transfer::Sdr); + CHECK(wide.bt2020); + CHECK(wide.NeedsConversion()); +} + +TEST_CASE("Tone curve keeps SDR levels and compresses highlights", "[libopenshot][color]") { + const float peak = 1000.0f / 203.0f; + CHECK(color::ToneMap(0.5f, peak) == 0.5f); + CHECK(color::ToneMap(color::kToneKnee, peak) == Approx(color::kToneKnee)); + CHECK(color::ToneMap(peak, peak) == Approx(1.0f)); + CHECK(color::ToneMap(2.0f * peak, peak) == 1.0f); + CHECK(color::ToneMap(3.0f, 1.0f) == 1.0f); + float previous = 0.0f; + for (float value = 0.0f; value <= peak; value += 0.01f) { + const float mapped = color::ToneMap(value, peak); + CHECK(mapped >= previous); + previous = mapped; + } + // Reference white stays bright, leaving headroom for highlights. + CHECK(color::ToneMap(1.0f, peak) > 0.85f); + CHECK(color::ToneMap(1.0f, peak) < 1.0f); +} + +TEST_CASE("CPU decoding converts SDR, HLG and PQ charts to SDR BT.709", "[libopenshot][color][ffmpegreader]") { + SoftwareDecode software; + const float hdr_peak = float(color::kDefaultPeakNits / color::kReferenceWhiteNits); + CheckChart("color-chart-sdr-bt709.mp4", 1.0f, 4); + CheckChart("color-chart-hlg.mp4", hdr_peak, 4); + CheckChart("color-chart-pq.mp4", hdr_peak, 4); +} + +TEST_CASE("Exports are tagged and round trip their colors", "[libopenshot][color][ffmpegwriter]") { + if (!avcodec_find_encoder_by_name("libx264")) { + WARN("libx264 unavailable; skipping"); + return; + } + SoftwareDecode software; + struct Size { int width, height; AVColorSpace matrix; }; + for (const Size size : {Size{1280, 720, AVCOL_SPC_BT709}, Size{640, 480, AVCOL_SPC_SMPTE170M}}) { + QTemporaryDir directory; + REQUIRE(directory.isValid()); + const std::string path = directory.filePath("color.mp4").toStdString(); + const char* colors[] = {"#dc1e1e", "#28c828", "#1e28dc", "#e0ac8c"}; + { + FFmpegWriter writer(path); + writer.SetVideoOptions(true, "libx264", Fraction(30, 1), size.width, size.height, + Fraction(1, 1), false, false, 20000000); + writer.Open(); + for (int number = 1; number <= 4; ++number) + writer.WriteFrame(std::make_shared(number, size.width, size.height, colors[number - 1])); + writer.Close(); + } + + AVFormatContext* input = nullptr; + REQUIRE(avformat_open_input(&input, path.c_str(), nullptr, nullptr) == 0); + REQUIRE(avformat_find_stream_info(input, nullptr) >= 0); + const int index = av_find_best_stream(input, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + REQUIRE(index >= 0); + const AVCodecParameters* parameters = input->streams[index]->codecpar; + CHECK(parameters->color_space == size.matrix); + CHECK(parameters->color_primaries == AVCOL_PRI_BT709); + CHECK(parameters->color_trc == AVCOL_TRC_BT709); + CHECK(parameters->color_range == AVCOL_RANGE_MPEG); + avformat_close_input(&input); + + FFmpegReader reader(path); + reader.Open(); + for (int number = 1; number <= 4; ++number) { + const QColor expected(colors[number - 1]); + const QColor actual = QColor::fromRgba(reader.GetFrame(number)->GetImage()->pixel(size.width / 2, size.height / 2)); + INFO(size.width << "x" << size.height << " " << colors[number - 1] << " got " << actual.name().toStdString()); + CHECK(std::abs(actual.red() - expected.red()) <= 4); + CHECK(std::abs(actual.green() - expected.green()) <= 4); + CHECK(std::abs(actual.blue() - expected.blue()) <= 4); + } + reader.Close(); + } +} diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 5171db1ff..3ddfaed70 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -1,5 +1,6 @@ #ifdef __APPLE__ #include "MacVulkanInterop.h" +#include #endif // Copyright (c) 2026 OpenShot Studios, LLC // SPDX-License-Identifier: LGPL-3.0-or-later @@ -7,6 +8,9 @@ #include "VulkanCompositor.h" #include "VulkanFrameImage.h" #include "VulkanEncodeConverter.h" +#include "ColorManagement.h" +#include "Settings.h" +#include #include "GpuDevices.h" #include "GpuMetrics.h" #include "FrameScope.h" @@ -352,7 +356,7 @@ TEST_CASE("Vulkan affine surfaces preserve alpha color and ownership", "[vulkan] CHECK(blended.alpha()==255); CHECK(image.pixelColor(0,0)==QColor(Qt::blue)); CHECK(second->Materialize().pixelColor(0,0)==QColor(Qt::red)); - // Export conversion remains on the GPU and produces limited-range BT.601 NV12. + // Export conversion remains on the GPU and produces limited-range NV12 (BT.601 at this SD size). auto encoded = openshot::ConvertVulkanToNV12(second); auto encoded_surface = std::dynamic_pointer_cast(encoded); REQUIRE(encoded_surface); @@ -500,10 +504,10 @@ TEST_CASE("Vulkan high quality reader resizing uses bicubic antialiasing", "[vul uint8_t* output[]={reference.bits(),nullptr,nullptr,nullptr}; int strides[]={reference.bytesPerLine(),0,0,0}; REQUIRE(sws_scale(scaler,cpu->data,cpu->linesize,0,32,output,strides)==size.height()); - auto high=processor.Process(gpu.get(),size.width(),size.height(),true); + auto high=processor.Process(gpu.get(),size.width(),size.height(),openshot::color::SourceColor(),true); REQUIRE(std::dynamic_pointer_cast(high)); auto high_image=high->Materialize(); - auto low_image=processor.Process(gpu.get(),size.width(),size.height(),false)->Materialize(); + auto low_image=processor.Process(gpu.get(),size.width(),size.height(),openshot::color::SourceColor(),false)->Materialize(); double high_error=0,low_error=0; for(int y=0;yHARDWARE_DECODER; + ~DecoderSetting(){openshot::Settings::Instance()->HARDWARE_DECODER=previous;} + } restore; + // Mean of a flat patch interior; see tests/generate-color-charts.py. + auto patch=[](std::shared_ptr frame,int index){ + const int x0=(index%4)*80+30,y0=(index/4)*48+18; + std::array sum{}; + for(int y=y0;yGetPixels(y)[x*4+c]; + return std::array{int(sum[0]/240),int(sum[1]/240),int(sum[2]/240)}; + }; + for(const char* chart : {"color-chart-sdr-bt709.mp4","color-chart-hlg.mp4","color-chart-pq.mp4"}) { + const std::string path=std::string(TEST_MEDIA_PATH)+chart; + // The CPU pipeline is selected per reader when it opens. + setenv("OPENSHOT_RENDER_BACKEND","cpu",1); + openshot::FFmpegReader cpu_reader(path); + cpu_reader.Open(); + unsetenv("OPENSHOT_RENDER_BACKEND"); + auto cpu=cpu_reader.GetFrame(2); + REQUIRE_FALSE(cpu->GetDeferredImage()); + openshot::Settings::Instance()->HARDWARE_DECODER=1; + openshot::FFmpegReader gpu_reader(path); + gpu_reader.Open(); + auto gpu=gpu_reader.GetFrame(2); + // 10-bit P010 surfaces stay on the GPU instead of falling back to software decode. +#ifdef __APPLE__ + // VideoToolbox decodes H.264 and HEVC Main10 on every supported Mac. + REQUIRE(gpu->GetDeferredImage()); +#else + if(!gpu->GetDeferredImage()) { + WARN("Skipping Vulkan test: this device does not decode " << chart << " with Vulkan"); + continue; + } +#endif + for(int i=0;i<13;++i) { + const auto expected=patch(cpu,i),actual=patch(gpu,i); + INFO(chart << " patch " << i << " cpu (" << expected[0] << "," << expected[1] << "," << expected[2] + << ") gpu (" << actual[0] << "," << actual[1] << "," << actual[2] << ")"); + for(int c=0;c<3;++c) CHECK(std::abs(actual[c]-expected[c])<=3); + } + } +} + +TEST_CASE("Vulkan export encodes and tags HD with BT.709", "[vulkan][hardware][color]") { + if(!VulkanAvailable()) return; + QImage red(1280,720,QImage::Format_RGBA8888_Premultiplied); + red.fill(QColor(255,0,0)); + auto encoded=std::dynamic_pointer_cast( + openshot::ConvertVulkanToNV12(openshot::UploadVulkanImage(red))); + REQUIRE(encoded); + const AVFrame* nv12=encoded->GetFrame(); + CHECK(nv12->colorspace==AVCOL_SPC_BT709); + CHECK(nv12->color_primaries==AVCOL_PRI_BT709); + CHECK(nv12->color_trc==AVCOL_TRC_BT709); + CHECK(nv12->color_range==AVCOL_RANGE_MPEG); + Frame downloaded(av_frame_alloc()); + REQUIRE(downloaded); +#ifdef __APPLE__ + REQUIRE(openshot::ReadMacVulkanFrame(downloaded.get(), nv12) >= 0); +#else + REQUIRE(av_hwframe_transfer_data(downloaded.get(), nv12, 0) >= 0); +#endif + // BT.709 limited range red: Y 63, Cb 102, Cr 240 (BT.601 would be 81, 90, 240). + const int y=downloaded->data[0][100*downloaded->linesize[0]+100]; + const int u=downloaded->data[1][50*downloaded->linesize[1]+100]; + const int v=downloaded->data[1][50*downloaded->linesize[1]+101]; + CHECK(y>=62); CHECK(y<=64); + CHECK(u>=101); CHECK(u<=103); + CHECK(v>=239); CHECK(v<=241); +} + +#ifdef __APPLE__ +namespace { +// A Core Video 4:2:0 biplanar buffer filled with one Y'CbCr value per half +// (left, right), as VideoToolbox returns decoded frames. 10-bit formats keep +// samples in the high bits of 16-bit words, like P010. +struct PixelBuffer { + CVPixelBufferRef pixel=nullptr; + PixelBuffer(OSType type,int width,int height,std::array left,std::array right,int bits) { + const void* keys[]={kCVPixelBufferIOSurfacePropertiesKey,kCVPixelBufferMetalCompatibilityKey}; + CFDictionaryRef empty=CFDictionaryCreate(kCFAllocatorDefault,nullptr,nullptr,0, + &kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks); + const void* values[]={empty,kCFBooleanTrue}; + CFDictionaryRef attributes=CFDictionaryCreate(kCFAllocatorDefault,keys,values,2, + &kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks); + const CVReturn created=CVPixelBufferCreate(kCFAllocatorDefault,width,height,type,attributes,&pixel); + CFRelease(attributes);CFRelease(empty); + REQUIRE(created==kCVReturnSuccess); + REQUIRE(CVPixelBufferLockBaseAddress(pixel,0)==kCVReturnSuccess); + const int shift=bits>8?16-bits:0; + auto store=[&](uint8_t* row,int x,int value){ + if(bits>8) reinterpret_cast(row)[x]=uint16_t(value<(CVPixelBufferGetBaseAddressOfPlane(pixel,plane)); + const size_t stride=CVPixelBufferGetBytesPerRowOfPlane(pixel,plane); + const int plane_width=int(CVPixelBufferGetWidthOfPlane(pixel,plane)); + const int plane_height=int(CVPixelBufferGetHeightOfPlane(pixel,plane)); + for(int y=0;yformat=AV_PIX_FMT_VIDEOTOOLBOX; + frame->width=int(CVPixelBufferGetWidth(pixel));frame->height=int(CVPixelBufferGetHeight(pixel)); + frame->data[3]=reinterpret_cast(pixel); + frame->colorspace=matrix;frame->color_trc=trc;frame->color_primaries=primaries; + frame->color_range=AVCOL_RANGE_MPEG; + return frame; + } +}; +} + +TEST_CASE("Mac VideoToolbox surfaces map to shareable FFmpeg layouts", "[vulkan][hardware][mac][color]") { + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)==AV_PIX_FMT_NV12); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)==AV_PIX_FMT_NV12); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange)==AV_PIX_FMT_P010); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange)==AV_PIX_FMT_P010); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange)==AV_PIX_FMT_P210); + CHECK(openshot::MacSurfaceFormat(kCVPixelFormatType_32BGRA)==AV_PIX_FMT_NONE); +} + +TEST_CASE("Mac 10-bit VideoToolbox surfaces convert on the GPU", "[vulkan][hardware][mac][color]") { + if(!VulkanAvailable()) return; + // Left half: BT.709 limited-range red (Y'CbCr 250, 409, 960 at 10 bits). + // Right half: 75% grey, the HLG reference white (Y' 721, neutral chroma). + const std::array red{250,409,960},grey{721,512,512}; + for(const OSType type : {kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange,kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange}) { + INFO("Core Video format " << type); + PixelBuffer buffer(type,64,32,red,grey,10); + openshot::VulkanFrameProcessor processor; + + // SDR BT.709: exercises 16-bit plane mapping and 10-bit scaling. + auto sdr=buffer.Wrap(AVCOL_SPC_BT709,AVCOL_TRC_BT709,AVCOL_PRI_BT709); + const auto sdr_color=openshot::color::DescribeFrame(sdr.get(),64,32); + auto surface=processor.Process(sdr.get(),64,32,sdr_color); + REQUIRE(std::dynamic_pointer_cast(surface)); + QImage image=surface->Materialize(); + const QColor left=image.pixelColor(8,16),right=image.pixelColor(56,16); + INFO("SDR left " << left.name().toStdString() << " right " << right.name().toStdString()); + CHECK(left.red()>=252);CHECK(left.green()<=3);CHECK(left.blue()<=3); + // 75% grey without HDR conversion: (721-64)/876 = 0.75 -> 191. + CHECK(std::abs(right.red()-191)<=2);CHECK(std::abs(right.green()-191)<=2);CHECK(std::abs(right.blue()-191)<=2); + + // HLG BT.2020: reference white maps to the tone curve's SDR white. + auto hlg=buffer.Wrap(AVCOL_SPC_BT2020_NCL,AVCOL_TRC_ARIB_STD_B67,AVCOL_PRI_BT2020); + const auto hlg_color=openshot::color::DescribeFrame(hlg.get(),64,32); + REQUIRE(hlg_color.NeedsConversion()); + image=processor.Process(hlg.get(),64,32,hlg_color)->Materialize(); + const int white=int(std::lround(255.0*std::pow(openshot::color::ToneMap(1.0f,hlg_color.RelativePeak()),1.0/2.4))); + const QColor hlg_grey=image.pixelColor(56,16); + INFO("HLG grey " << hlg_grey.name().toStdString() << " expected " << white); + CHECK(std::abs(hlg_grey.red()-white)<=3);CHECK(std::abs(hlg_grey.green()-white)<=3);CHECK(std::abs(hlg_grey.blue()-white)<=3); + + // CPU readback of the shared surface keeps the 10-bit samples. + Frame imported(openshot::ImportVideoToolboxFrame(sdr.get())); + REQUIRE(imported); + Frame readback(av_frame_alloc()); + REQUIRE(readback); + REQUIRE(openshot::ReadMacVulkanFrame(readback.get(),imported.get())>=0); + CHECK(readback->format==(type==kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange?AV_PIX_FMT_P010:AV_PIX_FMT_P210)); + CHECK(reinterpret_cast(readback->data[0]+16*readback->linesize[0])[8]==uint16_t(250<<6)); + CHECK(reinterpret_cast(readback->data[0]+16*readback->linesize[0])[56]==uint16_t(721<<6)); + } +} +#endif diff --git a/tests/generate-color-charts.py b/tests/generate-color-charts.py new file mode 100644 index 000000000..e82b8c8f1 --- /dev/null +++ b/tests/generate-color-charts.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Generate the color chart fixtures used by tests/ColorManagement.cpp. + +Each chart is a 4x4 grid of flat patches (80x48 px, 320x192 frame, 3 frames) +with known colors, (near-)losslessly encoded: + + examples/color-chart-sdr-bt709.mp4 8-bit H.264, BT.709 tagged + examples/color-chart-hlg.mp4 10-bit HEVC, BT.2020 + HLG (iPhone style) + examples/color-chart-pq.mp4 10-bit HEVC, BT.2020 + PQ (HDR10) + +Patches are SDR code values (PATCHES in tests/ColorManagement.cpp). HDR charts +place them per ITU-R BT.2408: BT.1886 display light with 100% = 203 nits. The +last two patches are 600 and 1000 nit highlights. + +Usage: generate-color-charts.py [ffmpeg] [output_dir] (needs numpy, libx264, libx265) +""" +import subprocess +import sys + +import numpy as np + +FFMPEG = sys.argv[1] if len(sys.argv) > 1 else "ffmpeg" +OUT = sys.argv[2] if len(sys.argv) > 2 else "examples" +W, H, PW, PH = 320, 192, 80, 48 +PATCHES = [(255, 255, 255), (118, 118, 118), (224, 172, 140), (141, 85, 62), (98, 155, 210), + (87, 122, 59), (220, 30, 30), (40, 200, 40), (30, 40, 220), (240, 220, 40), + (20, 20, 20), 600.0, 1000.0] +WHITE = 203.0 +M709_TO_2020 = np.array([[0.6274, 0.3293, 0.0433], [0.0691, 0.9195, 0.0114], [0.0164, 0.0880, 0.8956]]) +K709, K2020 = (0.2126, 0.0722), (0.2627, 0.0593) + + +def pq(nits): + m1, m2, c1, c2, c3 = 2610 / 16384, 2523 / 4096 * 128, 3424 / 4096, 2413 / 4096 * 32, 2392 / 4096 * 32 + y = np.clip(nits / 10000.0, 0, 1) ** m1 + return ((c1 + c2 * y) / (1 + c3 * y)) ** m2 + + +def hlg(nits): # display light (BT.2020) -> HLG signal, 1000 nit display, gamma 1.2 + a, b, c = 0.17883277, 0.28466892, 0.55991073 + yd = nits @ np.array([K2020[0], 1 - sum(K2020), K2020[1]]) + ys = (np.maximum(yd, 1e-9) / 1000.0) ** (1 / 1.2) + e = np.clip(nits / 1000.0 / np.maximum(ys ** 0.2, 1e-9)[..., None], 0, 1) + return np.where(e <= 1 / 12, np.sqrt(3 * e), a * np.log(np.maximum(12 * e - b, 1e-12)) + c) + + +def chart_nits(): + image = np.zeros((H, W, 3)) + for i, patch in enumerate(PATCHES): + r, c = divmod(i, 4) + value = np.full(3, patch) if isinstance(patch, float) else (np.array(patch) / 255.0) ** 2.4 * WHITE + image[r * PH:(r + 1) * PH, c * PW:(c + 1) * PW] = value + return image + + +def write(name, rgb, k, bits, tags, encoder): + kr, kb = k + y = rgb @ np.array([kr, 1 - kr - kb, kb]) + cb, cr = (rgb[..., 2] - y) / (2 * (1 - kb)), (rgb[..., 0] - y) / (2 * (1 - kr)) + s = 1 << (bits - 8) + planes = np.stack([16 + 219 * y, 128 + 224 * cb, 128 + 224 * cr]) * s + planes = np.round(planes).clip(0, (1 << bits) - 1).astype(np.uint16 if bits > 8 else np.uint8) + pix_in = "yuv444p10le" if bits > 8 else "yuv444p" + pix_out = "yuv420p10le" if bits > 8 else "yuv420p" + subprocess.run([FFMPEG, "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", pix_in, "-s", f"{W}x{H}", + "-r", "30", *tags, "-i", "-", *tags, "-pix_fmt", pix_out, *encoder, f"{OUT}/{name}"], + input=planes.tobytes() * 3, check=True) + + +def tags(trc, primaries, matrix): + return ["-color_primaries", primaries, "-color_trc", trc, "-colorspace", matrix, "-color_range", "tv"] + + +nits = chart_nits() +# Near-lossless standard Main10 (as cameras produce; some hardware decoders +# reject lossless HEVC). +x265 = ["-c:v", "libx265", "-x265-params", "crf=1:log-level=error", "-tag:v", "hvc1"] +write("color-chart-sdr-bt709.mp4", np.clip(nits / WHITE, 0, 1) ** (1 / 2.4), K709, 8, + tags("bt709", "bt709", "bt709"), ["-c:v", "libx264", "-profile:v", "high", "-crf", "1"]) +write("color-chart-hlg.mp4", hlg(nits @ M709_TO_2020.T), K2020, 10, + tags("arib-std-b67", "bt2020", "bt2020nc"), x265) +write("color-chart-pq.mp4", pq(nits @ M709_TO_2020.T), K2020, 10, + tags("smpte2084", "bt2020", "bt2020nc"), x265) From 1dd96277394b5b99a65772610088a3d313e97788 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 20:32:22 -0600 Subject: [PATCH 46/50] Fix color tests on Windows and Intel Mac CI - Set OPENSHOT_RENDER_BACKEND with qputenv/qunsetenv; MinGW has no setenv/unsetenv. - Skip charts the hardware decoder declines on every platform. VideoToolbox on Intel Macs rejects the 320x192 H.264 chart; the Mac 10-bit surface path is covered by the synthetic CVPixelBuffer tests. --- tests/ColorManagement.cpp | 13 ++++++++----- tests/VulkanCompositor.cpp | 16 +++++++--------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/ColorManagement.cpp b/tests/ColorManagement.cpp index e03c15b1c..cbc4055da 100644 --- a/tests/ColorManagement.cpp +++ b/tests/ColorManagement.cpp @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include "openshot_catch.h" @@ -87,21 +89,22 @@ void CheckChart(const std::string& file, float relative_peak, int tolerance) { struct SoftwareDecode { int previous = Settings::Instance()->HARDWARE_DECODER; #ifdef OPENSHOT_ENABLE_GPU_FRAME - std::string backend; + QByteArray backend; bool had_backend = false; #endif SoftwareDecode() { Settings::Instance()->HARDWARE_DECODER = 0; #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (const char* value = std::getenv("OPENSHOT_RENDER_BACKEND")) { backend = value; had_backend = true; } - setenv("OPENSHOT_RENDER_BACKEND", "cpu", 1); + had_backend = qEnvironmentVariableIsSet("OPENSHOT_RENDER_BACKEND"); + backend = qgetenv("OPENSHOT_RENDER_BACKEND"); + qputenv("OPENSHOT_RENDER_BACKEND", "cpu"); #endif } ~SoftwareDecode() { Settings::Instance()->HARDWARE_DECODER = previous; #ifdef OPENSHOT_ENABLE_GPU_FRAME - if (had_backend) setenv("OPENSHOT_RENDER_BACKEND", backend.c_str(), 1); - else unsetenv("OPENSHOT_RENDER_BACKEND"); + if (had_backend) qputenv("OPENSHOT_RENDER_BACKEND", backend); + else qunsetenv("OPENSHOT_RENDER_BACKEND"); #endif } }; diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 3ddfaed70..d7164b9bc 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -1229,26 +1229,24 @@ TEST_CASE("Vulkan decoding converts SDR, HLG and PQ charts like the CPU path", " for(const char* chart : {"color-chart-sdr-bt709.mp4","color-chart-hlg.mp4","color-chart-pq.mp4"}) { const std::string path=std::string(TEST_MEDIA_PATH)+chart; // The CPU pipeline is selected per reader when it opens. - setenv("OPENSHOT_RENDER_BACKEND","cpu",1); + qputenv("OPENSHOT_RENDER_BACKEND","cpu"); openshot::FFmpegReader cpu_reader(path); cpu_reader.Open(); - unsetenv("OPENSHOT_RENDER_BACKEND"); + qunsetenv("OPENSHOT_RENDER_BACKEND"); auto cpu=cpu_reader.GetFrame(2); REQUIRE_FALSE(cpu->GetDeferredImage()); openshot::Settings::Instance()->HARDWARE_DECODER=1; openshot::FFmpegReader gpu_reader(path); gpu_reader.Open(); auto gpu=gpu_reader.GetFrame(2); - // 10-bit P010 surfaces stay on the GPU instead of falling back to software decode. -#ifdef __APPLE__ - // VideoToolbox decodes H.264 and HEVC Main10 on every supported Mac. - REQUIRE(gpu->GetDeferredImage()); -#else + // Hardware decoders may decline these small charts (VideoToolbox on an + // Intel Mac rejects the 320x192 H.264 one) and FFmpeg then decodes in + // software. Mac 10-bit surfaces are covered with synthetic + // CVPixelBuffers below. if(!gpu->GetDeferredImage()) { - WARN("Skipping Vulkan test: this device does not decode " << chart << " with Vulkan"); + WARN("Skipping Vulkan test: this device does not decode " << chart << " on the GPU"); continue; } -#endif for(int i=0;i<13;++i) { const auto expected=patch(cpu,i),actual=patch(gpu,i); INFO(chart << " patch " << i << " cpu (" << expected[0] << "," << expected[1] << "," << expected[2] From 8d7baf9d5d91c3b8345cbb7bdc444bcf403b4f4f Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 20:48:41 -0600 Subject: [PATCH 47/50] Enlarge color charts to 640x384 so Intel Macs decode them on the GPU VideoToolbox on an Intel Mac (UHD 630) refuses 320x192 H.264 with "VideoToolbox session not available" (FFmpeg requires its hardware decoder for H.264), while 640x360 and larger decode. Regenerate the charts at 640x384 and require GPU decoding of them on macOS again. --- examples/color-chart-hlg.mp4 | Bin 3930 -> 4283 bytes examples/color-chart-pq.mp4 | Bin 3809 -> 4206 bytes examples/color-chart-sdr-bt709.mp4 | Bin 2352 -> 2442 bytes tests/ColorManagement.cpp | 8 ++++---- tests/VulkanCompositor.cpp | 13 ++++++++----- tests/generate-color-charts.py | 4 ++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/examples/color-chart-hlg.mp4 b/examples/color-chart-hlg.mp4 index 3f20b43750151a4279a32154972e2b0d9d540533..cd8183092b64dc5b9620f538fc3bea30de1e6b55 100644 GIT binary patch delta 1417 zcmV;41$O${9=josCVvFFZDe6|00006P63kv5`X{z1UV=HuLSc6J5SZBri4NDPOK~Y z&^V~3^{|}Tz8~;!nxOxVexRl${|~f56A5S>E`<8oW5exm3B$p7;@rUkrQ_B8LQ+qi z|KJtSGi>`{LEp-*kRE-yPpA3HG?&RhQ@7V4zkmP$4iixQS%2r3_r=%!z!*N??zT>? z|E#V600RI3NI)w_=fdPYMJmsy)DJ(N@s~_pzmd8+fn{*v{+tFjSUht2(W-#bxd@IJ zNWqmB2xj&5N?|1<%=-txTIQfWNuy25BA+$g#xFY6InHy?hC?^Yb_Ut=MCy;@JNm0ZsMs1!T&Ge{uxFI z(97$Wlii_QXKK-pw{3*?n}fX#v?eU?9FP8-m%^_RZcOM+R2Qp_ysxDXD~2fquL4S8 z5qPSX)fy$M5a(Vd2VYv}nB_Zu-eKkd5zhPW0LsD78-IxLzvt}`n|>=D?nWm$B}BP- z30n`p){%7(tO)r@(^ajKrKZufi|KLUN>xG!jZ8+Qm7Ii>;6FQ^jpJ>egO zgz)j1O*bVi6k4$eBUzPEAI6`kDp4fKWR>D|Klv5#V^Oap+|&~;(2MxKe~HPo+Du4M z@`Q;zQ7;2iAie=ubr}O0NuT%fQM*b^WQ3Xlxr` z4B+!Xea}E`t?wxRY?O7Y)rhax0`!tcxY}9)?ZY&w7!WsM#Fy1KWe;m&bU{>HmN@yf zPUmOe&{GqzLhLzKkrgk?XXe-E{$s`KvVYZIMpd83Wd4S?{U?+85?X{>z|D$`YbWd~ z&nn(YAPZtHU}>ugZ3QTc8D>dUFWH`Af|vfZfubBdCFdO6v#*c;|Nq+DG8YhFojqf; z_{Y>d(ckXau>e31*i>MNUC0s1#Ru6#ZGK-jqi_O2S_bj%>XrMEmMEBKb6Y?F0Dl7j z00RI30|8N9fB*oAoMExVhtPx$)I3ePqbuDHCQk?h&4AI-@vZ;#0fZ340DG(Z->KB} z-$0!Y_(~PTZV)3B0009300RI30{{R60009300smA001@u0niawgfW6~h|=h=TN#2# z0JwndAjgO1YbAm~Bb}TG8}NW)vPIr&Y%L&oAov&*@K!vx0e1iZ04)Fk;3U6>2<9R> znm}{CunRbm_DNn_sYXNq0DgvnFMeQqz_cjf6cBa7gbTO;01Mrdf&+5_2$S{$SOx-s z0004ilT-vL0f3W!1XKYllhXu08S8U&Vr&2a2{3bXb7TMj00000009612_Taz1sntd zfB}G$O9eF&nP_%nLjeLH0001x0000006&wG1tT^;m6!>k0096kLID8+AOHXZ0FVFx z0{{R606(Aw0s(2(^$~F!KUm*gr^M~y=^ZELk2H4Y#}f*Is@X|@!|euO zr$xM7UsnDe{l9PY065;Q8I75-{SbvV0_BN$K_kO?jh;GA!hf9*8*-h*b)L|2VEone zOtDxTFcPy*#-klP#L zVSkQ8;+>qpjfYpyGI`v<6yE$_!P34ozt(I6GuAv?&Hw!q$G$Dr%N7J-`0I0-4VF8d zr6>kWZQjN6v3~%q=+gYFG!gBbdsJASnk5T>Q{U6W4r%9(e;Pg`+1LKIBGOCSW_xL> zzPu_q+YIG4b{S9eSAxY#ryKQ$Gav4EyccKmJY4tXOAv%4B9o5c4%;6cP1O22SG0gu z)0l#y4y>np>uN(y4_f?Kiic_?<`>N`Z3HkWDA3^a_zd%r;ZGZOFNE^AjqTieLX*zd7QHjRe*aTi znRqdwNPiUIe{)wTl4PvulBk-+gEmOa=FLsRc{w8Q@cp6Wy3JSowj|q$B zTz@ef!D8U(W*JBOQuQC000Rub98fL0000000001000RelZyr%1OY$*z>}*6H4>O;c4I>U z0w4eY0FVFx0000ylkx^5H9VD=384T104zcQ0RkWZ00RJ!0009300RI#pb7{K6J?c} zNiw;pNRUV%00093AOHXZ1HrRP2TTD6GcquEF*!1`o(Ka60SS|$51I-C5&!@IBme*a I9Fs;6b<`Es9{>OV diff --git a/examples/color-chart-pq.mp4 b/examples/color-chart-pq.mp4 index d7045f9822298b064f2f89627a332c454dc902c4..ae79235d491c7552ebbf271dbbdc975f7b66865b 100644 GIT binary patch delta 1372 zcmY+DdpOg39LImZZFXrHN{&5!bI_{<&tiaeJST@+{)%$;-oV;|z|Extu) zCoLucUF2|7M;lX$$~>2JQItYk-=c`Gb2q3bg%SEbM+!YhTN_^W5xE4x&YHz1eQUkmJtLo<7uW@G_^s0$Q{>-1tTv_T^ocU*HcbeQPh z{yt!UyQ^LAr}niiZZH(Culb}zj&|b9tF&CXve*|wh^mMke+(UecsaGYoL>Li&G2B# zoQP$7KYBASVpK@xBD;3WDwBheKtWMs|8BP8(@;Se&4F|+O}Fxaw|}E}trCdz@%NLu zkVP(<;EPt6J7j(8c)h4v-_kN%7guXCvKSR1tgfG!62|}f=X{E?u<%wnjYO?_>D@fo z{PQ%Kc$F`W5I(pXD>LMNQ-=s(z27O5(4o?c8fm}9Iy10z$PfwV9J;4kBIsOcZ zTY_m9BcwytRC{W#S8P>F89j^6a6xPiyT{sS7;8!plUwaZ1xN~NFt1OrN1t0fvBfHc zwUydbM{4NIkE^XP)0wuXC8yijIa6r!AIs8NZVlKr{8oq7dQPebJDrVS?FB;f)8-;4KD}Sd~v4QPFW%a}!Vf*1K*6Ei$8Ba^eKb6uU%K)I5`3_}FIGny5+MrdTMOcK5r*bX*xg2`f1t) z@`@hAO<%cfI|gM1W|3byK6KxqsbBX_A@+HDo}Q|iex_K42s49Wc6Ce8-F4v8X%lmb z%ryImJAD~E-|p)4rlu_4w@3zI8MiZ1KQ@9(c(CW~&2D`cS-<%8>7AbCYz-GM8<+%l zY+wCS(Dw@F0RVs#4D8Vg;@D&+mtwMr^<%l1cD` z_suU9*0phZit9L6RdY`4dSE0mzwN41> z<@cq8x0r9kwiep|u<16PP1y;Xl^e4RRr=*xBrJxku*}OhP=Df(1pl=D{5Rs|%X>B~ zyVt)DklzU5KV>@*M}8RD*UNl+W!#L-@h#a%odb~mGW0$7N@=n*$nOWPjQ!`{%3{pz zu1_wH>_B#H9GALx8<#N5pKkeT$Mms}m4%q+ZP?@Swejf_@30dui~h`HLH~)Yg2TF^ zBdf3To_|8t`+wc#YsLW!$_Lg3+)y<#3EO$3ZN_^lKYkee|_x% zR=I+|`v1jZcKKArj zxHTCKMUeXhQ-);L-oORz1;#I-iUMJ-oGWt1ew3(?_-?ld(vd$p+ePZLewIG*JkUir z?{kamxqpHtB06V}t%0VU0T}IyY9Se=b8ITUaYKB@TRKW2_al7oP>&jZxEo;97<>L` zp>PB5|C93x+O8STOX4c>&BT_boeZ$_4!%SYPVM%4?mw690qKk;LO^o9?R2IiOWj;A z-!m40n!iV-5ZiYcFS;59!a$!NPJNC1T!~CWG=F$0Kbr2KyvS5`g)#i-UeH8AKH*vI zZJTtJ<{eS9f**GU)BpegAOZo<5m$sUf^#E0TJ4@Izz!nkQQat}gy}%aJHUW0`v3p{ z82|y`B)^6T?a6cW5PpC-C;$dau3&z?`S1V$3*2pQZ*~9x0Bmh`Xk-8Y0000000000 z0F$f)Tmc7@2?baN0YCr%0Kk)H1t|f4laU2f0VC000Rub98fL00000 z00001000RelRgF<1OY$*z>{7EH4>O;c4I>U0w4eY0FVFx0000ylcfeDH9VD=384T1 z04zcQ0RkWZ00RJ!0009300RI#pb7{K6J?c}Niw;pNPtKn00093AOHXZ1HrTY22245 mGcquEF*!1`Q3wJD0|4XDlU5Iz3Idn_001Na00146`44qh#Hd~X diff --git a/examples/color-chart-sdr-bt709.mp4 b/examples/color-chart-sdr-bt709.mp4 index cc8d12a53b059b97e1c287f0bd30745b29c85f47..2dfff15f9f27dafa69fbea300b4bfe409419b675 100644 GIT binary patch delta 1037 zcmV+o1oHc^5{eU$D-Ql`WMOmw00OQC1^@r5u{l%$7BMm)Y;SLCVQ6JxWM6b>a%Ev; zb3HP%q5(kxe*+k0h=c+E{`b_k%(CrMZ?+4l_oALAUqf{z0h=*7{etpSassNs7!3g7 z@&@RRQ8`v^{hU*t^Hq6+JG)pEQjLV+#qqYV*Vn$^umAuJ4VSJCb^YWW_yV~qhEcu= z^XsjM=7fM_WETJ(%>p?bRtm?CcJEG*ot@A2=c37je|(cJYv;h2?2k)7!P)cONT8j< z`hyd8%nIpSy%*ecj1o+SWb3B4w04A$txWU)FH;jh=Umgt_VF4*zcs-T>5Ss zybZ#*fAz$miF1b^_4~DwkEF#n8slt_cB|)(qNhy@BV=mPhtnDz21Oh3E7y#sZ6tnP zv80+M5t!DV%Dx+7Ac$C?iK*0z&GwE^fG4EboEpgUo7H+?|9me+ilSeAZ{B4p%RLwn6SFsZNm;;Gk&F z+K0L@bUO4wn@X&|w~=tq>wR@BpIlc9tn3m*k~AWKMhFl|HbBh%O2~H=Z!(X^Asx09 z=Wg$a4f41gR=4b^A~oc+qk48_|;c^_yn`2{6LglrgefyU03p!k-?pzZpj#VlHLfjP9EkH zp!SI^W9oi#`~dY1gqJ;#-@(nY!gOvde>kLjT=c^i3Sw#@4;^z&^>TB1Ye4EGFaQGM zSd2AD1D!rpf~@nu98l;*1~>lwEV8X8kd203YC}migSgR}9A_td4Ax|Sm?Htfr|l3L zK(F5X6N#;_3XK~q zQ+V8UtP1RL3c5!CO?t+ZkbnRH0{{R6000942><{95kZC0002Sb98fL00000000010002ClY0jo1Ok8ofRmdCH7Pk^c4I>UWB?xj z;Q$Cvd)hpv2v@)NVC1Xk2uTvi}^&dI8|7 zqfa$FMsE1Q)q)gSF+F~#qHNBVUZwuom4br$0yuBd)3S)k*G+F|?Fk`TndkssrVHA; z2;OfQ)73ggy3#_>s{hD~PgRSr(*OlYnrzscOs);Wxb?)KiF1b^_4~DwetucqQd?h| zPin${>MAiub>ca@!|9CXTdL{9kFh+9Mi6#b9Dqh^S~=OI00ei8T+ZklQ015$~L3w`w00RMw000AGZEtUO0001NZFXprH3nz_ zjgy`RSOx(=0002Mll%rL0R@vL2UG#DlY0k0IdF4yVr&2a0K{{2b7TMj0000000961 z0JLFtV=({#00000009610000000000000000003%0Kk*;2Q?=*VRmCf0b~FT|KR`{ zXJh~ktl2?9nrZ?90)PMj0|0;k03HX56w8xQ2rdG_V3UCeiwS@L000mG000h?6$y3a CpR^eO diff --git a/tests/ColorManagement.cpp b/tests/ColorManagement.cpp index cbc4055da..23d7ea0e7 100644 --- a/tests/ColorManagement.cpp +++ b/tests/ColorManagement.cpp @@ -30,7 +30,7 @@ using namespace openshot; namespace { // Patches of examples/color-chart-*.mp4 (see tests/generate-color-charts.py): -// SDR code values, then 600 and 1000 nit highlights. 4x4 grid of 80x48 patches. +// SDR code values, then 600 and 1000 nit highlights. 4x4 grid of 160x96 patches. struct Patch { const char* name; std::array code; double nits; }; const Patch kPatches[] = { {"reference white", {255, 255, 255}, 0}, {"18% grey", {118, 118, 118}, 0}, @@ -58,7 +58,7 @@ std::array Expected(const Patch& patch, float relative_peak) { } std::array PatchColor(std::shared_ptr frame, int index) { - const int x0 = (index % 4) * 80 + 30, y0 = (index / 4) * 48 + 18; + const int x0 = (index % 4) * 160 + 70, y0 = (index / 4) * 96 + 42; long sum[3] = {0, 0, 0}; for (int y = y0; y < y0 + 12; ++y) { const unsigned char* row = frame->GetPixels(y); @@ -72,8 +72,8 @@ void CheckChart(const std::string& file, float relative_peak, int tolerance) { FFmpegReader reader(TEST_MEDIA_PATH + file); reader.Open(); auto frame = reader.GetFrame(2); - REQUIRE(frame->GetWidth() == 320); - REQUIRE(frame->GetHeight() == 192); + REQUIRE(frame->GetWidth() == 640); + REQUIRE(frame->GetHeight() == 384); for (int i = 0; i < int(std::size(kPatches)); ++i) { const auto expected = Expected(kPatches[i], relative_peak); const auto actual = PatchColor(frame, i); diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index d7164b9bc..f4a2bc28f 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -1220,7 +1220,7 @@ TEST_CASE("Vulkan decoding converts SDR, HLG and PQ charts like the CPU path", " } restore; // Mean of a flat patch interior; see tests/generate-color-charts.py. auto patch=[](std::shared_ptr frame,int index){ - const int x0=(index%4)*80+30,y0=(index/4)*48+18; + const int x0=(index%4)*160+70,y0=(index/4)*96+42; std::array sum{}; for(int y=y0;yGetPixels(y)[x*4+c]; @@ -1239,14 +1239,17 @@ TEST_CASE("Vulkan decoding converts SDR, HLG and PQ charts like the CPU path", " openshot::FFmpegReader gpu_reader(path); gpu_reader.Open(); auto gpu=gpu_reader.GetFrame(2); - // Hardware decoders may decline these small charts (VideoToolbox on an - // Intel Mac rejects the 320x192 H.264 one) and FFmpeg then decodes in - // software. Mac 10-bit surfaces are covered with synthetic - // CVPixelBuffers below. + // 10-bit P010 surfaces stay on the GPU instead of falling back to software decode. +#ifdef __APPLE__ + // VideoToolbox decodes H.264 and HEVC Main10 on every supported Mac. Intel + // Macs refuse very small H.264 frames (320x192), so the charts are 640x384. + REQUIRE(gpu->GetDeferredImage()); +#else if(!gpu->GetDeferredImage()) { WARN("Skipping Vulkan test: this device does not decode " << chart << " on the GPU"); continue; } +#endif for(int i=0;i<13;++i) { const auto expected=patch(cpu,i),actual=patch(gpu,i); INFO(chart << " patch " << i << " cpu (" << expected[0] << "," << expected[1] << "," << expected[2] diff --git a/tests/generate-color-charts.py b/tests/generate-color-charts.py index e82b8c8f1..c8aaa9ff7 100644 --- a/tests/generate-color-charts.py +++ b/tests/generate-color-charts.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: LGPL-3.0-or-later """Generate the color chart fixtures used by tests/ColorManagement.cpp. -Each chart is a 4x4 grid of flat patches (80x48 px, 320x192 frame, 3 frames) +Each chart is a 4x4 grid of flat patches (160x96 px, 640x384 frame, 3 frames) with known colors, (near-)losslessly encoded: examples/color-chart-sdr-bt709.mp4 8-bit H.264, BT.709 tagged @@ -23,7 +23,7 @@ FFMPEG = sys.argv[1] if len(sys.argv) > 1 else "ffmpeg" OUT = sys.argv[2] if len(sys.argv) > 2 else "examples" -W, H, PW, PH = 320, 192, 80, 48 +W, H, PW, PH = 640, 384, 160, 96 PATCHES = [(255, 255, 255), (118, 118, 118), (224, 172, 140), (141, 85, 62), (98, 155, 210), (87, 122, 59), (220, 30, 30), (40, 200, 40), (30, 40, 220), (240, 220, 40), (20, 20, 20), 600.0, 1000.0] From 338d9eaa481c7bdcfb9b362cb6068a97fb0714b2 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Fri, 6 Nov 2026 23:49:04 -0600 Subject: [PATCH 48/50] Load Vulkan at runtime and fall back to the CPU without it libopenshot no longer links libvulkan on Linux or Windows. It opens the Vulkan library on first use and loads only the functions it calls. When the library or a function is missing (headless servers, old systems without a GPU driver), OpenShot still starts and renders on the CPU. macOS keeps linking the bundled MoltenVK. --- src/CMakeLists.txt | 12 +++++-- src/GpuDevices.cpp | 5 +++ src/MacVulkanInterop.mm | 1 + src/Qt/VulkanOpenGLPresenter.cpp | 1 + src/VulkanCompositor.cpp | 1 + src/VulkanFrameImage.cpp | 3 +- src/VulkanLoader.cpp | 54 ++++++++++++++++++++++++++++++++ src/VulkanLoader.h | 47 +++++++++++++++++++++++++++ tests/VulkanCompositor.cpp | 4 +-- tests/VulkanDeviceLost.cpp | 2 +- tests/VulkanPresentation.cpp | 2 +- 11 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 src/VulkanLoader.cpp create mode 100644 src/VulkanLoader.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 406308d92..eb3db458a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -452,7 +452,7 @@ if(ENABLE_GPU_FRAME) list(APPEND all_comps avfilter) target_sources(openshot PRIVATE VulkanFrameImage.cpp) target_sources(openshot PRIVATE VulkanCompositor.cpp) - target_sources(openshot PRIVATE GpuDevices.cpp) + target_sources(openshot PRIVATE GpuDevices.cpp VulkanLoader.cpp) if(APPLE) enable_language(OBJCXX) target_sources(openshot PRIVATE MacVulkanInterop.mm MacVulkanInterop.h Qt/VulkanOpenGLPresenterMac.mm Qt/VulkanOpenGLPresenter.h) @@ -480,7 +480,15 @@ if(ENABLE_GPU_FRAME) add_custom_target(openshot-vulkan-shaders DEPENDS ${VULKAN_SHADER_HEADERS}) add_dependencies(openshot openshot-vulkan-shaders) target_include_directories(openshot PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/vulkan") - target_link_libraries(openshot PRIVATE Vulkan::Vulkan) + # Vulkan is loaded at runtime (VulkanLoader.cpp), so OpenShot starts without it. + # macOS links the MoltenVK it bundles; nothing to load there. + target_compile_definitions(openshot PRIVATE VK_NO_PROTOTYPES) + if(APPLE) + target_link_libraries(openshot PRIVATE Vulkan::Vulkan) + else() + target_include_directories(openshot PRIVATE ${Vulkan_INCLUDE_DIRS}) + target_link_libraries(openshot PRIVATE ${CMAKE_DL_LIBS}) + endif() find_package(PkgConfig QUIET) if(PKG_CONFIG_FOUND) pkg_check_modules(VULKAN_FRAME_PC QUIET vulkan) diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 0af127a96..09364673d 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -18,12 +18,15 @@ extern "C" { #include #include } +#include "VulkanLoader.h" namespace openshot { namespace { // A temporary instance for listing devices; FFmpeg keeps its own for rendering. struct ListingInstance { VkInstance instance = VK_NULL_HANDLE; ListingInstance() { + if (!LoadVulkan()) + return; VkApplicationInfo app = {}; app.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; app.apiVersion = VK_API_VERSION_1_1; @@ -215,6 +218,8 @@ int CreateSharedVulkanDevice(AVBufferRef** output) { if (shared.tried) return AVERROR(ENODEV); shared.tried = true; + if (!LoadVulkan()) + return AVERROR(ENODEV); #ifdef __APPLE__ // Work around imported-texture argument-buffer writes on older Intel // Metal drivers. An explicit user setting still takes precedence. diff --git a/src/MacVulkanInterop.mm b/src/MacVulkanInterop.mm index 92d4ebc3c..ed7e6a594 100644 --- a/src/MacVulkanInterop.mm +++ b/src/MacVulkanInterop.mm @@ -21,6 +21,7 @@ #include #include } +#include "VulkanLoader.h" namespace openshot { namespace { const char native_pool_tag = 0; diff --git a/src/Qt/VulkanOpenGLPresenter.cpp b/src/Qt/VulkanOpenGLPresenter.cpp index 79a21364d..752476a63 100644 --- a/src/Qt/VulkanOpenGLPresenter.cpp +++ b/src/Qt/VulkanOpenGLPresenter.cpp @@ -36,6 +36,7 @@ extern "C" { #include } +#include "../VulkanLoader.h" namespace openshot { namespace { diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 81dd7941c..6e3917070 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -25,6 +25,7 @@ extern "C" { #include #include } +#include "VulkanLoader.h" // SPIR-V generated from src/vulkan/*.comp at build time. #include "affine.comp.h" #include "encode_nv12.comp.h" diff --git a/src/VulkanFrameImage.cpp b/src/VulkanFrameImage.cpp index f2b646458..601ae22b9 100644 --- a/src/VulkanFrameImage.cpp +++ b/src/VulkanFrameImage.cpp @@ -28,6 +28,7 @@ extern "C" { #include #include } +#include "VulkanLoader.h" namespace openshot { namespace { void Check(int result, const char* operation) { @@ -84,7 +85,7 @@ void RecordVulkanUploadFailure() { } bool VulkanRenderingEnabled() { const char* backend = std::getenv("OPENSHOT_RENDER_BACKEND"); - return !(backend && std::strcmp(backend, "cpu") == 0) && !VulkanDeviceLost(); + return !(backend && std::strcmp(backend, "cpu") == 0) && !VulkanDeviceLost() && LoadVulkan(); } struct VulkanFrameImage::Impl { OwnedFrame frame; diff --git a/src/VulkanLoader.cpp b/src/VulkanLoader.cpp new file mode 100644 index 000000000..dcba4d08e --- /dev/null +++ b/src/VulkanLoader.cpp @@ -0,0 +1,54 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +#include "VulkanLoader.h" +#include "Logger.h" +#include +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#else +#include +#endif + +#define OPENSHOT_VULKAN_DEFINE(name) PFN_##name name = nullptr; +OPENSHOT_VULKAN_FUNCTIONS(OPENSHOT_VULKAN_DEFINE) +#undef OPENSHOT_VULKAN_DEFINE + +namespace openshot { +namespace { +bool Load() { + // Never unloaded: drivers keep threads and state for the process lifetime. +#if defined(_WIN32) + HMODULE library = LoadLibraryExA("vulkan-1.dll", nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + const auto symbol = [library](const char* name) { return reinterpret_cast(GetProcAddress(library, name)); }; +#elif defined(__APPLE__) + // MoltenVK ships inside the app and is linked directly + void* library = RTLD_DEFAULT; + const auto symbol = [](const char* name) { return dlsym(RTLD_DEFAULT, name); }; +#else + void* library = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); + const auto symbol = [library](const char* name) { return dlsym(library, name); }; +#endif + if (!library) { + Logger::Instance()->Log("Vulkan library not found; rendering on the CPU", Logger::LevelWarning); + return false; + } + bool complete = true; +#define OPENSHOT_VULKAN_LOAD(name) \ + name = reinterpret_cast(symbol(#name)); \ + if (!name) { \ + Logger::Instance()->Log("Vulkan library lacks " #name "; rendering on the CPU", Logger::LevelWarning); \ + complete = false; \ + } + OPENSHOT_VULKAN_FUNCTIONS(OPENSHOT_VULKAN_LOAD) +#undef OPENSHOT_VULKAN_LOAD + return complete; +} +} // namespace + +bool LoadVulkan() { + static const bool loaded = Load(); + return loaded; +} +} // namespace openshot diff --git a/src/VulkanLoader.h b/src/VulkanLoader.h new file mode 100644 index 000000000..50b1ef957 --- /dev/null +++ b/src/VulkanLoader.h @@ -0,0 +1,47 @@ +// Copyright (c) 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later +// +// Loads the Vulkan functions OpenShot uses at runtime, so libopenshot starts on +// systems without a Vulkan loader or driver (they render on the CPU instead). +// Build with VK_NO_PROTOTYPES; each pointer below has the same name as the +// Vulkan function, so calling code is unchanged. +#pragma once +#include + +#define OPENSHOT_VULKAN_FUNCTIONS(X) \ + X(vkAllocateCommandBuffers) X(vkAllocateDescriptorSets) X(vkAllocateMemory) \ + X(vkBeginCommandBuffer) X(vkBindBufferMemory) X(vkCmdBindDescriptorSets) \ + X(vkCmdBindPipeline) X(vkCmdCopyBuffer) X(vkCmdCopyImageToBuffer) X(vkCmdDispatch) \ + X(vkCmdFillBuffer) X(vkCmdPipelineBarrier) X(vkCmdPushConstants) X(vkCreateBuffer) \ + X(vkCreateCommandPool) X(vkCreateComputePipelines) X(vkCreateDescriptorPool) \ + X(vkCreateDescriptorSetLayout) X(vkCreateFence) X(vkCreateImage) X(vkCreateImageView) \ + X(vkCreateInstance) X(vkCreatePipelineLayout) X(vkCreateSampler) X(vkCreateSemaphore) \ + X(vkCreateShaderModule) X(vkDestroyBuffer) X(vkDestroyCommandPool) \ + X(vkDestroyDescriptorPool) X(vkDestroyDescriptorSetLayout) X(vkDestroyFence) \ + X(vkDestroyImage) X(vkDestroyImageView) X(vkDestroyInstance) X(vkDestroyPipeline) \ + X(vkDestroyPipelineLayout) X(vkDestroySampler) X(vkDestroySemaphore) \ + X(vkDestroyShaderModule) X(vkDeviceWaitIdle) X(vkEndCommandBuffer) \ + X(vkEnumerateDeviceExtensionProperties) X(vkEnumerateInstanceVersion) \ + X(vkEnumeratePhysicalDevices) X(vkFlushMappedMemoryRanges) X(vkFreeMemory) \ + X(vkGetBufferMemoryRequirements) X(vkGetBufferMemoryRequirements2) \ + X(vkGetDeviceProcAddr) X(vkGetDeviceQueue) X(vkGetPhysicalDeviceMemoryProperties) \ + X(vkGetPhysicalDeviceProperties) X(vkGetPhysicalDeviceProperties2) \ + X(vkGetPhysicalDeviceQueueFamilyProperties) X(vkInvalidateMappedMemoryRanges) \ + X(vkMapMemory) X(vkQueueSubmit) X(vkResetFences) X(vkUnmapMemory) \ + X(vkUpdateDescriptorSets) X(vkWaitForFences) X(vkWaitSemaphores) + +// Hidden, so other libraries in the process never bind to these pointers +#if defined(_WIN32) +#define OPENSHOT_VULKAN_HIDDEN +#else +#define OPENSHOT_VULKAN_HIDDEN __attribute__((visibility("hidden"))) +#endif +#define OPENSHOT_VULKAN_DECLARE(name) extern OPENSHOT_VULKAN_HIDDEN PFN_##name name; +OPENSHOT_VULKAN_FUNCTIONS(OPENSHOT_VULKAN_DECLARE) +#undef OPENSHOT_VULKAN_DECLARE + +namespace openshot { +// True when the Vulkan library and every function above were found. Cheap +// after the first call; nothing may call Vulkan when it returns false. +bool LoadVulkan(); +} diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index f4a2bc28f..9614b9061 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -48,7 +48,7 @@ using Frame=std::unique_ptr; // CTest reports this message as skipped; see tests/CMakeLists.txt. bool VulkanAvailable() { if(!openshot::VulkanRenderingEnabled()) { - WARN("Skipping Vulkan test: OPENSHOT_RENDER_BACKEND=cpu"); + WARN("Skipping Vulkan test: CPU mode, or no Vulkan library"); return false; } AVBufferRef* device=nullptr; @@ -713,7 +713,7 @@ TEST_CASE("Saved GPU preferences reach the native runtime environment", "[vulkan CHECK(std::string(std::getenv("OPENSHOT_VULKAN_DEVICE")) == "Example GPU"); CHECK_FALSE(openshot::VulkanRenderingEnabled()); openshot::ConfigureGpuEnvironment("vulkan", ""); - CHECK(openshot::VulkanRenderingEnabled()); + CHECK(std::string(std::getenv("OPENSHOT_RENDER_BACKEND")) == "vulkan"); CHECK(std::getenv("OPENSHOT_VULKAN_DEVICE") == nullptr); } diff --git a/tests/VulkanDeviceLost.cpp b/tests/VulkanDeviceLost.cpp index 11368e95e..81b37a9d7 100644 --- a/tests/VulkanDeviceLost.cpp +++ b/tests/VulkanDeviceLost.cpp @@ -12,7 +12,7 @@ extern "C" { TEST_CASE("A lost GPU device moves open readers and new work to the CPU", "[vulkan][recovery]") { if (!openshot::VulkanRenderingEnabled()) { - WARN("Skipping Vulkan test: OPENSHOT_RENDER_BACKEND=cpu"); + WARN("Skipping Vulkan test: CPU mode, or no Vulkan library"); return; } AVBufferRef* device = nullptr; diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 4fdb9caf2..359ca948e 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -93,7 +93,7 @@ struct Frames { TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[vulkan][presentation]") { // CTest reports "Skipping Vulkan test:" as skipped; see tests/CMakeLists.txt. if (!openshot::VulkanRenderingEnabled()) { - WARN("Skipping Vulkan test: OPENSHOT_RENDER_BACKEND=cpu"); + WARN("Skipping Vulkan test: CPU mode, or no Vulkan library"); return; } const QByteArray platform = qgetenv("QT_QPA_PLATFORM"); From b17d27e69ae01fdec104d2814776c97eb9d3f179 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 8 Nov 2026 14:22:35 -0600 Subject: [PATCH 49/50] Fix Vulkan lint warnings and missing license headers --- ci/linux/setup-ffmpeg-vulkan.sh | 2 + ci/patch-ffmpeg-vulkan.sh | 2 + .../ffmpeg-vulkan-readback.patch.license | 2 + ci/windows/setup-ffmpeg-windows-final.sh | 2 + ci/windows/setup-ffmpeg-windows.sh | 2 + ci/windows/setup-hw-ffmpeg-windows.sh | 2 + ci/windows/smoke-ffmpeg-windows.sh | 2 + examples/run-timeline-benchmarks.py | 167 ++++++++++-------- src/GpuDevices.cpp | 5 +- src/GpuMetrics.cpp | 1 + src/GpuMetrics.h | 1 + src/Qt/ExternalOpenGL.h | 1 + src/VulkanCompositor.cpp | 11 +- src/vulkan/affine.comp | 2 + src/vulkan/encode_nv12.comp | 2 + tests/GpuMetrics.cpp | 1 + tests/VulkanCompositor.cpp | 7 +- tests/generate-color-charts.py | 9 +- 18 files changed, 139 insertions(+), 82 deletions(-) create mode 100644 ci/patches/ffmpeg-vulkan-readback.patch.license diff --git a/ci/linux/setup-ffmpeg-vulkan.sh b/ci/linux/setup-ffmpeg-vulkan.sh index 80d4c5008..fa56a020c 100755 --- a/ci/linux/setup-ffmpeg-vulkan.sh +++ b/ci/linux/setup-ffmpeg-vulkan.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later # Install FFmpeg with Vulkan (plus the codecs the focal AppImage ships) into # /opt, without touching the system FFmpeg used by the CPU-only builds. # Ubuntu 20.04 (focal). Run as a sudo-capable user. Safe to re-run. diff --git a/ci/patch-ffmpeg-vulkan.sh b/ci/patch-ffmpeg-vulkan.sh index 3155759ae..55b3faaea 100644 --- a/ci/patch-ffmpeg-vulkan.sh +++ b/ci/patch-ffmpeg-vulkan.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later # Apply the FFmpeg 7.1 Vulkan host-import and staging-copy fixes. set -euo pipefail script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) diff --git a/ci/patches/ffmpeg-vulkan-readback.patch.license b/ci/patches/ffmpeg-vulkan-readback.patch.license new file mode 100644 index 000000000..e8cd53c24 --- /dev/null +++ b/ci/patches/ffmpeg-vulkan-readback.patch.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +SPDX-License-Identifier: LGPL-3.0-or-later diff --git a/ci/windows/setup-ffmpeg-windows-final.sh b/ci/windows/setup-ffmpeg-windows-final.sh index a65e7ec86..9ab7699a9 100755 --- a/ci/windows/setup-ffmpeg-windows-final.sh +++ b/ci/windows/setup-ffmpeg-windows-final.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later # Run using C:\msys64\usr\bin\bash.exe. No pacman changes. # Usage: bash setup-ffmpeg-windows.sh mingw64|mingw32 set -euo pipefail diff --git a/ci/windows/setup-ffmpeg-windows.sh b/ci/windows/setup-ffmpeg-windows.sh index 8b3646a44..c915b545b 100755 --- a/ci/windows/setup-ffmpeg-windows.sh +++ b/ci/windows/setup-ffmpeg-windows.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later set -euo pipefail arch=${1:?Specify mingw64 or mingw32} export PATH="/$arch/bin:/usr/bin:/c/Windows/System32:/c/Windows" diff --git a/ci/windows/setup-hw-ffmpeg-windows.sh b/ci/windows/setup-hw-ffmpeg-windows.sh index c42531e2d..37592c7b2 100755 --- a/ci/windows/setup-hw-ffmpeg-windows.sh +++ b/ci/windows/setup-hw-ffmpeg-windows.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later # Build pinned hardware encoder dependencies without changing the installed toolchain. # Run with C:\msys64\usr\bin\bash.exe setup-hw-ffmpeg-windows.sh mingw64|mingw32. set -euo pipefail diff --git a/ci/windows/smoke-ffmpeg-windows.sh b/ci/windows/smoke-ffmpeg-windows.sh index 8db1f12cb..a98357e1c 100755 --- a/ci/windows/smoke-ffmpeg-windows.sh +++ b/ci/windows/smoke-ffmpeg-windows.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later set -euo pipefail arch=${1:?mingw64 or mingw32} prefix=${PREFIX:-/c/runner-setup/ffmpeg-7.1.3-$arch} diff --git a/examples/run-timeline-benchmarks.py b/examples/run-timeline-benchmarks.py index b3bd6018d..a3fffc862 100644 --- a/examples/run-timeline-benchmarks.py +++ b/examples/run-timeline-benchmarks.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Run comparable CPU/GPU timeline workloads sequentially and retain raw evidence. +# SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +# SPDX-License-Identifier: LGPL-3.0-or-later +""" +Run comparable CPU/GPU timeline workloads sequentially and retain raw evidence. Example (inherit the Vulkan ICD and FFmpeg library environment): python3 examples/run-timeline-benchmarks.py --cpu-build build --vulkan-build build-vk \ @@ -15,7 +18,8 @@ import re import shutil import statistics -import subprocess +# Local developer tools use explicit argument lists, never a shell. +import subprocess # nosec B404 import sys RESULT = re.compile( @@ -26,13 +30,101 @@ def capture(command, timeout=20): try: - result = subprocess.run(command, text=True, stdout=subprocess.PIPE, + # Developer-selected local tools run with argument lists, never a shell. + result = subprocess.run(command, text=True, stdout=subprocess.PIPE, # nosec B603 stderr=subprocess.STDOUT, timeout=timeout) return {"command": command, "returncode": result.returncode, "output": result.stdout} except (OSError, subprocess.TimeoutExpired) as error: return {"command": command, "error": str(error)} +def validate_result(row, frames, layers, readback): + """Reject incomplete runs and unexpected CPU fallback before recording FPS.""" + match = RESULT.search(row.get("output", "")) + if row.get("returncode") == 0 and match: + got_frames, got_layers, got_readback, deferred, seconds, fps = match.groups() + row.update(frames=int(got_frames), deferred_frames=int(deferred), + seconds=float(seconds), fps=float(fps)) + row["valid"] = (int(got_frames) == frames and int(got_layers) == layers + and int(got_readback) == readback and float(seconds) > 0 + and (row["backend"] != "vulkan" or int(deferred) == frames)) + if not row["valid"]: + row["error"] = "Incomplete workload or Vulkan CPU fallback detected" + else: + row["valid"] = False + + +def verify_image(row, image_path, cpu_images, layers, readback): + """Compare a GPU result with the preceding CPU trial of the same workload.""" + from PIL import Image, ImageChops, ImageStat + backend = row["backend"] + try: + with Image.open(image_path) as loaded: + pixels = loaded.convert("RGB") + means = ImageStat.Stat(pixels).mean + row["image"] = {"path": str(image_path), "size": pixels.size, "mean_rgb": means} + if backend == "cpu": + cpu_images[(layers, readback)] = pixels + else: + reference = cpu_images[(layers, readback)] + if pixels.size != reference.size: + raise ValueError("GPU image dimensions differ from CPU") + row["image"]["cpu_mae_rgb"] = ImageStat.Stat(ImageChops.difference(pixels, reference)).mean + reference_means = ImageStat.Stat(reference).mean + if sum(reference_means) > 3 and sum(means) < 0.1 * sum(reference_means): + raise ValueError("GPU image is black/near-black while matching CPU image is not") + except (OSError, ValueError) as error: + row["valid"] = False + row["error"] = str(error) + + +def summarize(backend, layers, readback, successes): + """Keep failed trials out of the aggregate measurements.""" + return {"backend": backend, "layers": layers, "readback": readback, + "valid_runs": len(successes), + "median_fps": statistics.median(successes) if successes else None, + "min_fps": min(successes) if successes else None, + "max_fps": max(successes) if successes else None} + + +def measure(args, binaries, video, report, output): + """Run sequential trials, retaining CPU images for the GPU comparisons.""" + def save(): + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2) + "\n") + + if args.verify_images: + image_dir = output.parent / (output.stem + "-frames") + image_dir.mkdir(parents=True, exist_ok=True) + cpu_images = {} + failed = False + for layers in (1, 2): + for readback in (0, 1): + for backend, binary in binaries.items(): + successes = [] + for trial in range(1, args.runs + 1): + command = [str(binary), str(video), str(args.frames), str(layers), str(readback)] + image_path = None + if args.verify_images: + image_path = image_dir / f"{backend}-layers{layers}-readback{readback}-run{trial}.png" + command.append(str(image_path)) + evidence = capture(command, args.timeout) + row = {"backend": backend, "layers": layers, "readback": readback, + "trial": trial, **evidence} + validate_result(row, args.frames, layers, readback) + if row["valid"] and image_path: + verify_image(row, image_path, cpu_images, layers, readback) + failed |= not row["valid"] + if row["valid"]: + successes.append(row["fps"]) + report["measurements"].append(row) + print(f"{backend} layers={layers} readback={readback} trial={trial}: " + f"{row.get('fps', 'no result')} FPS valid={row['valid']}", flush=True) + save() + report["summary"].append(summarize(backend, layers, readback, successes)) + return failed + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--cpu-build", required=True, type=Path) @@ -83,75 +175,10 @@ def main(): if shutil.which("nvidia-smi"): report["gpu_processes"] = capture(["nvidia-smi", "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader"]) - def save(): - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(report, indent=2) + "\n") - - if args.verify_images: - from PIL import Image, ImageChops, ImageStat - image_dir = output.parent / (output.stem + "-frames") - image_dir.mkdir(parents=True, exist_ok=True) - cpu_images = {} - failed = False - for layers in (1, 2): - for readback in (0, 1): - for backend, binary in binaries.items(): - successes = [] - for trial in range(1, args.runs + 1): - command = [str(binary), str(video), str(args.frames), str(layers), str(readback)] - image_path = None - if args.verify_images: - image_path = image_dir / f"{backend}-layers{layers}-readback{readback}-run{trial}.png" - command.append(str(image_path)) - evidence = capture(command, args.timeout) - row = {"backend": backend, "layers": layers, "readback": readback, - "trial": trial, **evidence} - match = RESULT.search(evidence.get("output", "")) - if evidence.get("returncode") == 0 and match: - frames, got_layers, got_readback, deferred, seconds, fps = match.groups() - row.update(frames=int(frames), deferred_frames=int(deferred), - seconds=float(seconds), fps=float(fps)) - row["valid"] = (int(frames) == args.frames and int(got_layers) == layers - and int(got_readback) == readback and float(seconds) > 0 - and (backend != "vulkan" or int(deferred) == args.frames)) - if not row["valid"]: - row["error"] = "Incomplete workload or Vulkan CPU fallback detected" - else: - row["valid"] = False - if row["valid"] and image_path: - try: - with Image.open(image_path) as loaded: - pixels = loaded.convert("RGB") - means = ImageStat.Stat(pixels).mean - row["image"] = {"path": str(image_path), "size": pixels.size, "mean_rgb": means} - if backend == "cpu": - cpu_images[(layers, readback)] = pixels - else: - reference = cpu_images[(layers, readback)] - if pixels.size != reference.size: - raise ValueError("GPU image dimensions differ from CPU") - row["image"]["cpu_mae_rgb"] = ImageStat.Stat(ImageChops.difference(pixels, reference)).mean - reference_means = ImageStat.Stat(reference).mean - if sum(reference_means) > 3 and sum(means) < 0.1 * sum(reference_means): - raise ValueError("GPU image is black/near-black while matching CPU image is not") - except (OSError, ValueError) as error: - row["valid"] = False - row["error"] = str(error) - failed |= not row["valid"] - if row["valid"]: - successes.append(row["fps"]) - report["measurements"].append(row) - print(f"{backend} layers={layers} readback={readback} trial={trial}: " - f"{row.get('fps', 'no result')} FPS valid={row['valid']}", flush=True) - save() - report["summary"].append({"backend": backend, "layers": layers, "readback": readback, - "valid_runs": len(successes), - "median_fps": statistics.median(successes) if successes else None, - "min_fps": min(successes) if successes else None, - "max_fps": max(successes) if successes else None}) + failed = measure(args, binaries, video, report, output) report["finished_utc"] = datetime.datetime.now(datetime.timezone.utc).isoformat() report["valid"] = not failed - save() + output.write_text(json.dumps(report, indent=2) + "\n") print(f"Results: {output}") return 1 if failed else 0 diff --git a/src/GpuDevices.cpp b/src/GpuDevices.cpp index 09364673d..48bcd432d 100644 --- a/src/GpuDevices.cpp +++ b/src/GpuDevices.cpp @@ -4,6 +4,7 @@ #include "Logger.h" #include "VulkanFrameImage.h" #include +#include #include #include #include @@ -143,8 +144,8 @@ std::vector VulkanDeviceCandidates() { std::stable_sort(ranked.begin(), ranked.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); std::vector candidates; - for (const auto& device : ranked) - candidates.push_back(device.second); + std::transform(ranked.begin(), ranked.end(), std::back_inserter(candidates), + [](const auto& device) { return device.second; }); return candidates; } diff --git a/src/GpuMetrics.cpp b/src/GpuMetrics.cpp index d9a625a4d..5539c6f28 100644 --- a/src/GpuMetrics.cpp +++ b/src/GpuMetrics.cpp @@ -1,3 +1,4 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC // SPDX-License-Identifier: LGPL-3.0-or-later #include "GpuMetrics.h" #include "Frame.h" diff --git a/src/GpuMetrics.h b/src/GpuMetrics.h index e83977de2..2a255fa57 100644 --- a/src/GpuMetrics.h +++ b/src/GpuMetrics.h @@ -1,3 +1,4 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC // SPDX-License-Identifier: LGPL-3.0-or-later #ifndef OPENSHOT_GPU_METRICS_H #define OPENSHOT_GPU_METRICS_H diff --git a/src/Qt/ExternalOpenGL.h b/src/Qt/ExternalOpenGL.h index 928190c83..14296e97f 100644 --- a/src/Qt/ExternalOpenGL.h +++ b/src/Qt/ExternalOpenGL.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT /* ** Copyright (c) 2013-2018 The Khronos Group Inc. ** diff --git a/src/VulkanCompositor.cpp b/src/VulkanCompositor.cpp index 6e3917070..b4cba575e 100644 --- a/src/VulkanCompositor.cpp +++ b/src/VulkanCompositor.cpp @@ -9,6 +9,7 @@ #include "MacVulkanInterop.h" #endif #include +#include #include #include #include @@ -208,7 +209,7 @@ class Pipeline { VkCheck(to_device?vkFlushMappedMemoryRanges(device,1,&range):vkInvalidateMappedMemoryRanges(device,1,&range),"Synchronize shader buffer"); } public: - Pipeline(AVBufferRef* device_owner,bool encode=false):encode_nv12(encode) { + explicit Pipeline(AVBufferRef* device_owner,bool encode=false):encode_nv12(encode) { device_ref=av_buffer_ref(device_owner); if(!device_ref) throw std::bad_alloc(); device_context=reinterpret_cast(device_ref->data); @@ -433,7 +434,7 @@ class Pipeline { Sync(slot.data,true); slot.data_owner=effect->data_owner; } - const VkDeviceSize count_bytes=scope?effect->counts->size()*sizeof(uint32_t):0; + const VkDeviceSize count_bytes=effect && effect->counts?effect->counts->size()*sizeof(uint32_t):0; if(scope) { Reserve(slot.counts,count_bytes,VK_BUFFER_USAGE_STORAGE_BUFFER_BIT|VK_BUFFER_USAGE_TRANSFER_SRC_BIT|VK_BUFFER_USAGE_TRANSFER_DST_BIT, 0,VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); @@ -447,13 +448,13 @@ class Pipeline { all.push_back(extra); std::sort(all.begin(),all.end(),[](const AVFrame* a,const AVFrame* b){return std::less()(a->data[0],b->data[0]);}); std::vector> locks; - for(auto* f:all) locks.emplace_back(new LockedFrame(f)); + std::transform(all.begin(),all.end(),std::back_inserter(locks), + [](const AVFrame* f){return std::make_unique(f);}); auto* sf=reinterpret_cast(src->data[0]); auto* of=reinterpret_cast(output->data[0]); const bool separate_nv12 = p.modes[0]==1 && sf->img[1]!=VK_NULL_HANDLE && sf->img[1]!=sf->img[0]; if(separate_nv12) plane_count=2; const auto* svc=reinterpret_cast(src_fc->hwctx); - auto& owned_views=slot.views; VkCommandBuffer cmd=slot.cmd; const VkDescriptorSet set=slot.set; try { @@ -469,7 +470,7 @@ class Pipeline { } std::array views{}; auto make_view=[&](VkImage image,VkFormat format,VkImageAspectFlags aspect){ - auto view=View(image,format,aspect);owned_views.push_back(view);return view; + auto view=View(image,format,aspect);slot.views.push_back(view);return view; }; if(p.modes[0]==1){ views[0]=make_view(sf->img[0],wide?VK_FORMAT_R16_UNORM:VK_FORMAT_R8_UNORM,separate_nv12?VK_IMAGE_ASPECT_COLOR_BIT:VK_IMAGE_ASPECT_PLANE_0_BIT); diff --git a/src/vulkan/affine.comp b/src/vulkan/affine.comp index 9b8a9ce5d..3d54ca364 100644 --- a/src/vulkan/affine.comp +++ b/src/vulkan/affine.comp @@ -1,3 +1,5 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later #version 450 layout(local_size_x=16, local_size_y=16) in; layout(binding=0) uniform sampler2D plane_y; diff --git a/src/vulkan/encode_nv12.comp b/src/vulkan/encode_nv12.comp index 91375fa2d..b323a78cd 100644 --- a/src/vulkan/encode_nv12.comp +++ b/src/vulkan/encode_nv12.comp @@ -1,3 +1,5 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC +// SPDX-License-Identifier: LGPL-3.0-or-later #version 450 layout(local_size_x=16,local_size_y=16) in; layout(binding=0) uniform sampler2D source_image; diff --git a/tests/GpuMetrics.cpp b/tests/GpuMetrics.cpp index 93502c879..10d28ad8d 100644 --- a/tests/GpuMetrics.cpp +++ b/tests/GpuMetrics.cpp @@ -1,3 +1,4 @@ +// SPDX-FileCopyrightText: 2026 OpenShot Studios, LLC // SPDX-License-Identifier: LGPL-3.0-or-later #include "openshot_catch.h" #include "GpuMetrics.h" diff --git a/tests/VulkanCompositor.cpp b/tests/VulkanCompositor.cpp index 9614b9061..2d5b530ee 100644 --- a/tests/VulkanCompositor.cpp +++ b/tests/VulkanCompositor.cpp @@ -18,6 +18,7 @@ #include "FFmpegWriter.h" #include #include +#include #include "Clip.h" #include "effects/Saturation.h" #include "effects/Crop.h" @@ -568,7 +569,9 @@ TEST_CASE("VideoToolbox export shares decoder compositor and encoder surfaces", CHECK(verify.info.video_length==24); auto image=verify.GetFrame(12)->GetImage(); REQUIRE(image); CHECK(image->width()==width); CHECK(image->height()==height); + REQUIRE(expected_frame); const auto expected = expected_frame->GetImage(); + REQUIRE(expected); double error = 0, brightness = 0; int samples = 0; for (int y = 8; y < height; y += 16) { @@ -1130,8 +1133,8 @@ TEST_CASE("Masked clip grading with a LUT and scopes stays on Vulkan", "[vulkan] scope.SetVideoRegionNormalized(0.25f,0.25f,0.5f,0.5f); scope.SetFrame(frame); REQUIRE(scope.HasVideo()); - uint64_t counted=0; - for (auto bin : scope.GetVideoHistogramLumaBins()) counted+=bin; + const auto bins=scope.GetVideoHistogramLumaBins(); + const uint64_t counted=std::accumulate(bins.begin(),bins.end(),uint64_t{0}); CHECK(counted>0); } // A neutral grade applies only its LUT, still on the GPU. diff --git a/tests/generate-color-charts.py b/tests/generate-color-charts.py index c8aaa9ff7..f3a169d10 100644 --- a/tests/generate-color-charts.py +++ b/tests/generate-color-charts.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # Copyright (c) 2026 OpenShot Studios, LLC # SPDX-License-Identifier: LGPL-3.0-or-later -"""Generate the color chart fixtures used by tests/ColorManagement.cpp. +""" +Generate the color chart fixtures used by tests/ColorManagement.cpp. Each chart is a 4x4 grid of flat patches (160x96 px, 640x384 frame, 3 frames) with known colors, (near-)losslessly encoded: @@ -16,7 +17,8 @@ Usage: generate-color-charts.py [ffmpeg] [output_dir] (needs numpy, libx264, libx265) """ -import subprocess +# Local developer tools use explicit argument lists, never a shell. +import subprocess # nosec B404 import sys import numpy as np @@ -64,8 +66,9 @@ def write(name, rgb, k, bits, tags, encoder): planes = np.round(planes).clip(0, (1 << bits) - 1).astype(np.uint16 if bits > 8 else np.uint8) pix_in = "yuv444p10le" if bits > 8 else "yuv444p" pix_out = "yuv420p10le" if bits > 8 else "yuv420p" + # The developer selects FFmpeg; all encoding options are fixed arguments. subprocess.run([FFMPEG, "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", pix_in, "-s", f"{W}x{H}", - "-r", "30", *tags, "-i", "-", *tags, "-pix_fmt", pix_out, *encoder, f"{OUT}/{name}"], + "-r", "30", *tags, "-i", "-", *tags, "-pix_fmt", pix_out, *encoder, f"{OUT}/{name}"], # nosec B603 input=planes.tobytes() * 3, check=True) From 16ccf9189389f3f9126795ed3ea3386055266c87 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Sun, 8 Nov 2026 15:42:16 -0600 Subject: [PATCH 50/50] Speed up waveform extraction and extract all channels in one pass Audio-only readers sized their frame cache from video dimensions, so the cache never evicted and every insert rescanned every cached frame. Long waveform reads became quadratic: 100 s for a 2 hour file, now 6 s. - Size audio-only FFmpegReader caches by audio, and keep the cache of readers owned by AudioWaveformer tiny (reads are strictly sequential) - Add ExtractChannelSamples() to decode the mixed envelope and every channel's envelope in a single pass - Place chunk boundaries exactly, so 44.1 kHz audio no longer drifts (16 s late after 2 hours) or loses the end of the file - Add an ApplyKeyframes() overload taking plain envelopes, interpolating keyframes within a frame and fixing a one frame offset --- src/AudioWaveformer.cpp | 383 ++++++++++++++++++++++++-------------- src/AudioWaveformer.h | 57 +++++- src/FFmpegReader.cpp | 8 + tests/AudioWaveformer.cpp | 270 +++++++++++++++++++++++++++ 4 files changed, 580 insertions(+), 138 deletions(-) diff --git a/src/AudioWaveformer.cpp b/src/AudioWaveformer.cpp index 3c0a5eb10..90437b6f2 100644 --- a/src/AudioWaveformer.cpp +++ b/src/AudioWaveformer.cpp @@ -31,6 +31,36 @@ using namespace std; using namespace openshot; +namespace { +// Waveform extraction reads every frame exactly once, front to back, so a large +// reader cache only costs memory (and, because CacheMemory sizes are rescanned on +// every insert, time). Keep readers owned by the waveformer nearly cache-free. +constexpr int64_t WAVEFORM_READER_CACHE_BYTES = 2 * 1024 * 1024; + +void LimitWaveformReaderCache(ReaderBase* reader) +{ + if (!reader) { + return; + } + if (CacheBase* cache = reader->GetCache()) { + cache->SetMaxBytes(WAVEFORM_READER_CACHE_BYTES); + } +} + +// Linearly interpolate a keyframe at a fractional (1-based) frame position +double KeyframeValueAt(const Keyframe& keyframe, double frame) +{ + const double floor_frame = std::floor(frame); + const int64_t index = static_cast(floor_frame); + const double value = keyframe.GetValue(index); + const double fraction = frame - floor_frame; + if (fraction <= 0.0) { + return value; + } + return value + (keyframe.GetValue(index + 1) - value) * fraction; +} +} + // Default constructor AudioWaveformer::AudioWaveformer(ReaderBase* new_reader) : reader(new_reader), @@ -81,9 +111,20 @@ AudioWaveformData AudioWaveformer::ExtractSamples(int channel, int num_per_secon return base; } +// Extract the mixed envelope plus one envelope per channel, in a single decode pass +AudioWaveformData AudioWaveformer::ExtractChannelSamples(int num_per_second, bool normalize) { + AudioWaveformData data; + if (!reader) { + return data; + } + ReaderBase* source = ResolveWaveformReader(); + return ExtractSamplesFromReader(source, -1, num_per_second, normalize, true); +} + AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize) { FFmpegReader temp_reader(path); temp_reader.Open(); + LimitWaveformReaderCache(&temp_reader); // Disable video for speed bool has_video = temp_reader.info.has_video; temp_reader.info.has_video = false; @@ -102,6 +143,7 @@ AudioWaveformData AudioWaveformer::ExtractSamples(const std::string& path, bool normalize) { FFmpegReader temp_reader(path); temp_reader.Open(); + LimitWaveformReaderCache(&temp_reader); bool has_video = temp_reader.info.has_video; temp_reader.info.has_video = false; Fraction source_fps = temp_reader.info.fps; @@ -121,96 +163,121 @@ AudioWaveformData AudioWaveformer::ApplyKeyframes(const AudioWaveformData& base, int channel, bool normalize) { AudioWaveformData data; - if (num_per_second <= 0) { + if (source_fps.ToDouble() <= 0.0) { return data; } + if (channel != -1 && (channel < 0 || channel >= source_channels)) { + return data; + } + + data = ApplyKeyframes(base.max_samples, base.rms_samples, time_keyframe, volume_keyframe, project_fps, num_per_second); + + if (normalize) { + float samples_max = 0.0f; + for (auto v : data.max_samples) { + samples_max = std::max(samples_max, std::abs(v)); + } + if (samples_max > 0.0f) { + data.scale(static_cast(data.max_samples.size()), 1.0f / samples_max); + } + } - double project_fps_value = project_fps.ToDouble(); - double source_fps_value = source_fps.ToDouble(); - if (project_fps_value <= 0.0 || source_fps_value <= 0.0) { + return data; +} + +AudioWaveformData AudioWaveformer::ApplyKeyframes(const std::vector& max_samples, + const std::vector& rms_samples, + const Keyframe* time_keyframe, + const Keyframe* volume_keyframe, + const Fraction& project_fps, + int num_per_second) { + AudioWaveformData data; + if (num_per_second <= 0) { return data; } - if (channel != -1 && (channel < 0 || channel >= source_channels)) { + const double project_fps_value = project_fps.ToDouble(); + if (project_fps_value <= 0.0) { return data; } - size_t base_total = base.max_samples.size(); + const size_t base_total = max_samples.size(); if (base_total == 0) { return data; } + const size_t rms_total = std::min(rms_samples.size(), base_total); - // Determine output duration from time curve (if any). Time curves are in project-frame domain. - int64_t output_frames = 0; - if (time_keyframe && time_keyframe->GetCount() > 0) { - output_frames = time_keyframe->GetLength(); + // A single point cannot remap time, and an empty volume curve has no gain to apply + if (time_keyframe && time_keyframe->GetCount() < 2) { + time_keyframe = nullptr; } - if (output_frames <= 0) { - // Default to source duration derived from base waveform length - double source_duration = static_cast(base_total) / static_cast(num_per_second); - output_frames = static_cast(std::llround(source_duration * project_fps_value)); + if (volume_keyframe && volume_keyframe->GetCount() < 1) { + volume_keyframe = nullptr; } - double output_duration_seconds = static_cast(output_frames) / project_fps_value; - int total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second)); + // Output duration follows the time curve (project frames), otherwise the source envelope + int64_t total_samples = static_cast(base_total); + if (time_keyframe) { + const int64_t output_frames = time_keyframe->GetLength(); + if (output_frames <= 0) { + return data; + } + const double output_duration_seconds = static_cast(output_frames) / project_fps_value; + total_samples = static_cast(std::ceil(output_duration_seconds * num_per_second)); + } if (total_samples <= 0) { return data; } - data.resize(total_samples); - data.zero(total_samples); + data.max_samples.assign(static_cast(total_samples), 0.0f); + data.rms_samples.assign(static_cast(total_samples), 0.0f); + + const double frames_per_sample = project_fps_value / static_cast(num_per_second); + const double samples_per_frame = static_cast(num_per_second) / project_fps_value; - for (int i = 0; i < total_samples; ++i) { - double out_time = static_cast(i) / static_cast(num_per_second); - // Time keyframes are defined in project-frame domain; evaluate using project frames - double project_frame = out_time * project_fps_value; - double mapped_project_frame = time_keyframe ? time_keyframe->GetValue(project_frame) : project_frame; - // Convert mapped project frame to seconds (project FPS), then to waveform index - double source_time = mapped_project_frame / project_fps_value; - double source_index = source_time * static_cast(num_per_second); + for (int64_t i = 0; i < total_samples; ++i) { + // 1-based project frame at this output sample (keyframes use 1-based frame numbers) + const double project_frame = static_cast(i) * frames_per_sample + 1.0; + + double source_index = static_cast(i); + if (time_keyframe) { + const double mapped_frame = KeyframeValueAt(*time_keyframe, project_frame); + source_index = (mapped_frame - 1.0) * samples_per_frame; + } // Sample base waveform (nearest with simple linear blend) - int idx0 = static_cast(std::floor(source_index)); - int idx1 = idx0 + 1; - double frac = source_index - static_cast(idx0); + const int64_t idx0 = static_cast(std::floor(source_index)); + const int64_t idx1 = idx0 + 1; + const double frac = source_index - static_cast(idx0); float max_sample = 0.0f; float rms_sample = 0.0f; - if (idx0 >= 0 && idx0 < static_cast(base_total)) { - max_sample = base.max_samples[idx0]; - rms_sample = base.rms_samples[idx0]; + if (idx0 >= 0 && idx0 < static_cast(base_total)) { + max_sample = max_samples[idx0]; + if (idx0 < static_cast(rms_total)) { + rms_sample = rms_samples[idx0]; + } } - if (idx1 >= 0 && idx1 < static_cast(base_total)) { - max_sample = static_cast((1.0 - frac) * max_sample + frac * base.max_samples[idx1]); - rms_sample = static_cast((1.0 - frac) * rms_sample + frac * base.rms_samples[idx1]); + if (frac > 0.0 && idx1 >= 0 && idx1 < static_cast(base_total)) { + max_sample = static_cast((1.0 - frac) * max_sample + frac * max_samples[idx1]); + if (idx1 < static_cast(rms_total)) { + rms_sample = static_cast((1.0 - frac) * rms_sample + frac * rms_samples[idx1]); + } } double gain = 1.0; if (volume_keyframe) { - double project_frame = out_time * project_fps_value; - gain = volume_keyframe->GetValue(project_frame); + gain = std::abs(KeyframeValueAt(*volume_keyframe, project_frame)); } - max_sample = static_cast(max_sample * gain); - rms_sample = static_cast(rms_sample * gain); - - data.max_samples[i] = max_sample; - data.rms_samples[i] = rms_sample; - } - if (normalize) { - float samples_max = 0.0f; - for (auto v : data.max_samples) { - samples_max = std::max(samples_max, std::abs(v)); - } - if (samples_max > 0.0f) { - data.scale(total_samples, 1.0f / samples_max); - } + data.max_samples[i] = static_cast(max_sample * gain); + data.rms_samples[i] = static_cast(rms_sample * gain); } return data; } -AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize) { +AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_reader, int channel, int num_per_second, bool normalize, bool per_channel) { AudioWaveformData data; if (!source_reader || num_per_second <= 0) { @@ -220,6 +287,9 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r // Open reader (if needed) if (!source_reader->IsOpen()) { source_reader->Open(); + if (source_reader == detached_reader.get()) { + LimitWaveformReaderCache(source_reader); + } } const auto retry_delay = std::chrono::milliseconds(100); @@ -245,14 +315,10 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r } }; - int sample_rate = source_reader->info.sample_rate; + int64_t sample_rate = source_reader->info.sample_rate; if (sample_rate <= 0) { sample_rate = num_per_second; } - int sample_divisor = sample_rate / num_per_second; - if (sample_divisor <= 0) { - sample_divisor = 1; - } // Determine length of video frames (for waveform) int64_t reader_video_length = source_reader->info.video_length; @@ -276,16 +342,21 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r return data; } - const bool known_duration = reader_duration > 0.0f && reader_video_length > 0; - int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0; - if (source_reader->info.channels == 0) { + const int reader_channels = source_reader->info.channels; + if (reader_channels <= 0) { return data; } - if (channel != -1 && (channel < 0 || channel >= source_reader->info.channels)) { + if (channel != -1 && (channel < 0 || channel >= reader_channels)) { return data; } + // Per-channel envelopes describe every source channel, so they need all channels + per_channel = per_channel && channel == -1; + + const bool known_duration = reader_duration > 0.0f && reader_video_length > 0; + const int total_samples = known_duration ? static_cast(std::ceil(reader_duration * num_per_second)) : 0; + // Resize and clear audio buffers when duration is known. Some audio-only // formats, especially raw FLAC, may not expose a container duration even // though frames are readable, so those are appended dynamically below. @@ -296,104 +367,146 @@ AudioWaveformData AudioWaveformer::ExtractSamplesFromReader(ReaderBase* source_r data.max_samples.reserve(num_per_second); data.rms_samples.reserve(num_per_second); } + if (per_channel) { + data.channel_max_samples.assign(reader_channels, std::vector(known_duration ? total_samples : 0, 0.0f)); + data.channel_rms_samples.assign(reader_channels, std::vector(known_duration ? total_samples : 0, 0.0f)); + } + + // Channels which contribute to the requested envelope + std::vector selected_channels; + for (int channel_index = 0; channel_index < reader_channels; channel_index++) { + if (channel == -1 || channel == channel_index) { + selected_channels.push_back(channel_index); + } + } + + // Per-channel accumulators for the current chunk + std::vector chunk_max(reader_channels, 0.0f); + std::vector chunk_squared_sum(reader_channels, 0.0); + std::vector channel_samples(reader_channels, nullptr); + + // Chunk k covers source samples [boundary(k - 1), boundary(k)). Boundaries are rounded + // from the exact rational position, so rates like 44100 / 200 = 220.5 never drift. + auto chunk_boundary = [&](int64_t chunk_index) -> int64_t { + return ((chunk_index + 1) * sample_rate + num_per_second / 2) / num_per_second; + }; int extracted_index = 0; - int sample_index = 0; + int64_t samples_read = 0; + int64_t chunk_sample_count = 0; + int64_t next_boundary = std::max(1, chunk_boundary(0)); float samples_max = 0.0f; - float chunk_max = 0.0f; - double chunk_squared_sum = 0.0; - int channel_count = (channel == -1) ? source_reader->info.channels : 1; - std::vector channels(source_reader->info.channels, nullptr); const double frame_rate = fps_value > 0.0 ? fps_value : 30.0; const int64_t max_unknown_frames = static_cast(std::ceil(frame_rate * 60.0 * 60.0 * 12.0)); int empty_audio_frames = 0; - auto append_sample = [&](float max_sample, float rms_sample) { - if (known_duration) { - if (extracted_index >= total_samples) { - return; - } - data.max_samples[extracted_index] = max_sample; - data.rms_samples[extracted_index] = rms_sample; - } else { - data.max_samples.push_back(max_sample); - data.rms_samples.push_back(rms_sample); - } - samples_max = std::max(samples_max, max_sample); - extracted_index++; + auto has_room = [&]() { + return !known_duration || extracted_index < total_samples; }; - try { - for (int64_t f = 1; - (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames); - f++) { - std::shared_ptr frame = get_frame_with_retry(f); - if (!known_duration && frame && frame->number < f) { - break; - } - int sample_count = frame->GetAudioSamplesCount(); - if (sample_count <= 0) { - if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) { - break; - } - continue; + // Store the current chunk as one envelope sample, then reset the accumulators + auto emit_chunk = [&]() { + if (chunk_sample_count <= 0) { + return; + } + if (has_room()) { + float mixed_max = 0.0f; + double mixed_squared_sum = 0.0; + for (int channel_index : selected_channels) { + mixed_max = std::max(mixed_max, chunk_max[channel_index]); + mixed_squared_sum += chunk_squared_sum[channel_index]; } - empty_audio_frames = 0; - - for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { - if (channel == channel_index || channel == -1) { - channels[channel_index] = frame->GetAudioSamples(channel_index); - } + const double mixed_count = static_cast(chunk_sample_count) * static_cast(selected_channels.size()); + const float mixed_rms = static_cast(std::sqrt(mixed_squared_sum / mixed_count)); + + if (known_duration) { + data.max_samples[extracted_index] = mixed_max; + data.rms_samples[extracted_index] = mixed_rms; + } else { + data.max_samples.push_back(mixed_max); + data.rms_samples.push_back(mixed_rms); } - - for (int s = 0; s < sample_count; s++) { - for (int channel_index = 0; channel_index < source_reader->info.channels; channel_index++) { - if (channel == channel_index || channel == -1) { - float *samples = channels[channel_index]; - if (!samples) { - continue; - } - float abs_sample = std::abs(samples[s]); - chunk_squared_sum += static_cast(samples[s]) * static_cast(samples[s]); - chunk_max = std::max(chunk_max, abs_sample); + if (per_channel) { + for (int channel_index = 0; channel_index < reader_channels; channel_index++) { + const float channel_rms = static_cast(std::sqrt( + chunk_squared_sum[channel_index] / static_cast(chunk_sample_count))); + if (known_duration) { + data.channel_max_samples[channel_index][extracted_index] = chunk_max[channel_index]; + data.channel_rms_samples[channel_index][extracted_index] = channel_rms; + } else { + data.channel_max_samples[channel_index].push_back(chunk_max[channel_index]); + data.channel_rms_samples[channel_index].push_back(channel_rms); } } + } + samples_max = std::max(samples_max, mixed_max); + extracted_index++; + } - sample_index += 1; + std::fill(chunk_max.begin(), chunk_max.end(), 0.0f); + std::fill(chunk_squared_sum.begin(), chunk_squared_sum.end(), 0.0); + chunk_sample_count = 0; + next_boundary = std::max(samples_read + 1, chunk_boundary(extracted_index)); + }; - if (sample_index % sample_divisor == 0) { - float avg_squared_sum = 0.0f; - if (channel_count > 0) { - avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_divisor * channel_count)); - } + for (int64_t f = 1; + (known_duration ? f <= reader_video_length && extracted_index < total_samples : f <= max_unknown_frames); + f++) { + std::shared_ptr frame = get_frame_with_retry(f); + if (!frame) { + break; + } + if (!known_duration && frame->number < f) { + break; + } + const int sample_count = frame->GetAudioSamplesCount(); + if (sample_count <= 0) { + if (!known_duration && extracted_index > 0 && ++empty_audio_frames >= 3) { + break; + } + continue; + } + empty_audio_frames = 0; - if (!known_duration || extracted_index < total_samples) { - append_sample(chunk_max, std::sqrt(avg_squared_sum)); - } + const int frame_channels = frame->GetAudioChannelsCount(); + for (int channel_index : selected_channels) { + channel_samples[channel_index] = channel_index < frame_channels ? frame->GetAudioSamples(channel_index) : nullptr; + } - sample_index = 0; - chunk_max = 0.0f; - chunk_squared_sum = 0.0; + int64_t offset = 0; + while (offset < sample_count) { + const int64_t span = std::min(sample_count - offset, next_boundary - samples_read); + for (int channel_index : selected_channels) { + const float* samples = channel_samples[channel_index]; + if (!samples) { + continue; + } + float span_max = chunk_max[channel_index]; + double span_squared_sum = chunk_squared_sum[channel_index]; + for (int64_t s = offset; s < offset + span; s++) { + const float value = samples[s]; + span_max = std::max(span_max, std::abs(value)); + span_squared_sum += static_cast(value) * static_cast(value); + } + chunk_max[channel_index] = span_max; + chunk_squared_sum[channel_index] = span_squared_sum; + } + offset += span; + samples_read += span; + chunk_sample_count += span; - if (known_duration && extracted_index >= total_samples) { - break; - } + if (samples_read >= next_boundary) { + emit_chunk(); + if (!has_room()) { + break; } } } - } catch (...) { - throw; } - if (sample_index > 0 && (!known_duration || extracted_index < total_samples)) { - float avg_squared_sum = 0.0f; - if (channel_count > 0) { - avg_squared_sum = static_cast(chunk_squared_sum / static_cast(sample_index * channel_count)); - } - - append_sample(chunk_max, std::sqrt(avg_squared_sum)); - } + // Keep the trailing partial chunk (end of the media) + emit_chunk(); if (normalize && samples_max > 0.0f) { float scale = 1.0f / samples_max; diff --git a/src/AudioWaveformer.h b/src/AudioWaveformer.h index 73ed9566f..73c591fa0 100644 --- a/src/AudioWaveformer.h +++ b/src/AudioWaveformer.h @@ -35,6 +35,10 @@ namespace openshot { std::vector max_samples; std::vector rms_samples; + /// Optional per-channel envelopes, filled by ExtractChannelSamples() + std::vector> channel_max_samples; + std::vector> channel_rms_samples; + /// Resize both datasets void resize(int total_samples) { max_samples.resize(total_samples); @@ -47,20 +51,30 @@ namespace openshot { std::fill(rms_samples.begin(), rms_samples.end(), 0); } - /// Scale # of values by some factor + /// Scale # of values by some factor (including any per-channel envelopes) void scale(int total_samples, float factor) { for (auto s = 0; s < total_samples; s++) { max_samples[s] *= factor; rms_samples[s] *= factor; } + for (auto& channel : channel_max_samples) { + for (auto& value : channel) value *= factor; + } + for (auto& channel : channel_rms_samples) { + for (auto& value : channel) value *= factor; + } } - /// Clear and free memory of both datasets + /// Clear and free memory of all datasets void clear() { max_samples.clear(); max_samples.shrink_to_fit(); rms_samples.clear(); rms_samples.shrink_to_fit(); + channel_max_samples.clear(); + channel_max_samples.shrink_to_fit(); + channel_rms_samples.clear(); + channel_rms_samples.shrink_to_fit(); } /// Return a vector of vectors (containing both datasets) @@ -70,6 +84,22 @@ namespace openshot { output.push_back(rms_samples); return output; } + + /// Number of per-channel envelopes available + int channel_count() const { + return static_cast(channel_max_samples.size()); + } + + /// Return the max and RMS envelopes of one channel (empty when unavailable) + std::vector> channel_vectors(int channel) const { + std::vector> output(2); + if (channel >= 0 && channel < channel_count() + && channel < static_cast(channel_rms_samples.size())) { + output[0] = channel_max_samples[channel]; + output[1] = channel_rms_samples[channel]; + } + return output; + } }; /** @@ -100,6 +130,16 @@ namespace openshot { /// @brief Extract audio samples from a media file path (audio-only fast path) AudioWaveformData ExtractSamples(const std::string& path, int channel, int num_per_second, bool normalize); + /// @brief Extract the mixed envelope and every channel's envelope in a single decode pass + /// + /// max_samples / rms_samples hold the all-channel envelope (the same values as + /// ExtractSamples(-1, ...)), and channel_max_samples / channel_rms_samples hold one + /// envelope per source channel (the same values as ExtractSamples(channel, ...)). + /// Clip keyframes are NOT applied, since per-channel envelopes describe the source media. + /// @param num_per_second How many samples per second to return + /// @param normalize Scale all envelopes by the largest mixed peak, so it becomes 1.0 + AudioWaveformData ExtractChannelSamples(int num_per_second, bool normalize); + /// @brief Apply time and volume keyframes to an existing waveform data set AudioWaveformData ApplyKeyframes(const AudioWaveformData& base, const openshot::Keyframe* time_keyframe, @@ -111,6 +151,17 @@ namespace openshot { int channel, bool normalize); + /// @brief Apply time and volume keyframes to existing max / RMS envelopes + /// + /// Keyframes are evaluated in the project-frame domain (1-based frames). Pass nullptr for + /// either keyframe to skip it. Time curves with fewer than 2 points are ignored. + AudioWaveformData ApplyKeyframes(const std::vector& max_samples, + const std::vector& rms_samples, + const openshot::Keyframe* time_keyframe, + const openshot::Keyframe* volume_keyframe, + const openshot::Fraction& project_fps, + int num_per_second); + /// @brief Convenience: extract then apply keyframes in one step from a file path AudioWaveformData ExtractSamples(const std::string& path, const openshot::Keyframe* time_keyframe, @@ -124,7 +175,7 @@ namespace openshot { ~AudioWaveformer(); private: - AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize); + AudioWaveformData ExtractSamplesFromReader(openshot::ReaderBase* source_reader, int channel, int num_per_second, bool normalize, bool per_channel = false); openshot::ReaderBase* ResolveSourceReader(openshot::ReaderBase* source_reader); openshot::Fraction ResolveSourceFPS(openshot::ReaderBase* source_reader); openshot::ReaderBase* ResolveWaveformReader(); diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp index 4c82a003d..55786c205 100644 --- a/src/FFmpegReader.cpp +++ b/src/FFmpegReader.cpp @@ -796,6 +796,14 @@ void FFmpegReader::Open() { const int final_cache_frames = std::max(Settings::Instance()->CACHE_MIN_FRAMES, OPEN_MP_NUM_PROCESSORS * 2); working_cache.SetMaxBytesFromInfo(working_cache_frames, info.width, info.height, info.sample_rate, info.channels); final_cache.SetMaxBytesFromInfo(final_cache_frames, info.width, info.height, info.sample_rate, info.channels); + if (!info.has_video && info.has_audio && info.sample_rate > 0 && info.channels > 0 && info.fps.ToDouble() > 0.0) { + // Audio-only frames carry no image, so a width x height budget is never + // reached. The cache would then keep every frame of the file, and since + // CacheMemory rescans its frames on each insert, long reads turn quadratic. + const int64_t samples_per_frame = static_cast(std::ceil(info.sample_rate / info.fps.ToDouble())); + const int64_t audio_frame_bytes = samples_per_frame * info.channels * static_cast(sizeof(float)); + final_cache.SetMaxBytes(final_cache_frames * audio_frame_bytes); + } #ifdef OPENSHOT_ENABLE_GPU_FRAME if (hw_de_on && hw_de_supported) { // Full-resolution hardware images consume device memory, not RAM. diff --git a/tests/AudioWaveformer.cpp b/tests/AudioWaveformer.cpp index 2ceac1ee4..de40a71c2 100644 --- a/tests/AudioWaveformer.cpp +++ b/tests/AudioWaveformer.cpp @@ -38,6 +38,49 @@ std::shared_ptr MakeWaveformFrame(int frame_number, int sample_rate, cons frame->AddAudio(true, 0, 0, samples.data(), static_cast(samples.size()), 1.0f); return frame; } + +// Write a stereo PCM WAV, where each channel's samples come from a generator(channel, sample_index) +template +void WriteStereoWav(const std::string& path, int sample_rate, double seconds, Generator generator) +{ + std::remove(path.c_str()); + FFmpegWriter writer(path); + writer.SetAudioOptions(true, "pcm_s16le", sample_rate, 2, LAYOUT_STEREO, sample_rate * 2 * 16); + writer.Open(); + + const int64_t total = static_cast(std::llround(seconds * sample_rate)); + const int chunk = sample_rate / 10; + int frame_number = 1; + for (int64_t start = 0; start < total; start += chunk) { + const int count = static_cast(std::min(chunk, total - start)); + auto frame = std::make_shared(frame_number++, count, 2); + frame->SampleRate(sample_rate); + frame->ChannelsLayout(LAYOUT_STEREO); + for (int channel = 0; channel < 2; channel++) { + std::vector samples(count); + for (int i = 0; i < count; i++) { + samples[i] = generator(channel, start + i); + } + frame->AddAudio(true, channel, 0, samples.data(), count, 1.0f); + } + writer.WriteFrame(frame); + } + writer.Close(); +} + +int ArgMax(const std::vector& values) +{ + return static_cast(std::distance(values.begin(), std::max_element(values.begin(), values.end()))); +} + +Keyframe LinearKeyframe(std::initializer_list> points) +{ + Keyframe keyframe; + for (const auto& point : points) { + keyframe.AddPoint(point.first, point.second, LINEAR); + } + return keyframe; +} } TEST_CASE( "Extract waveform data piano.wav", "[libopenshot][audiowaveformer]" ) @@ -507,3 +550,230 @@ TEST_CASE( "AudioWaveformData struct methods", "[libopenshot][audiowaveformer]" CHECK(vectors[0].size() == 0); CHECK(vectors[0].size() == 0); } + +TEST_CASE( "Waveform chunks do not drift when sample rate is not a multiple of the density", "[libopenshot][audiowaveformer][timing]" ) +{ + // 44100 / 200 = 220.5 samples per chunk. Integer chunking used 220 samples, which put + // this click 11 samples late at 25 seconds (and 16 seconds late at the end of 2 hours). + const std::string path = "AudioWaveformer-drift-44100.wav"; + const int sample_rate = 44100; + const double click_seconds = 25.0; + const int64_t click_start = static_cast(click_seconds * sample_rate); + WriteStereoWav(path, sample_rate, 30.0, [&](int, int64_t index) { + return (index >= click_start && index < click_start + 200) ? 0.9f : 0.001f; + }); + + FFmpegReader reader(path); + reader.Open(); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 200; + AudioWaveformData waveform = waveformer.ExtractSamples(-1, samples_per_second, false); + + REQUIRE(waveform.max_samples.size() == static_cast(std::ceil(reader.info.duration * samples_per_second))); + CHECK(ArgMax(waveform.max_samples) == static_cast(click_seconds * samples_per_second)); + + // The end of the media is covered (no truncated tail from accumulated rounding) + CHECK(waveform.max_samples.back() > 0.0f); + CHECK(waveform.rms_samples.back() > 0.0f); + + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE( "Single-pass channel extraction matches per-channel extraction", "[libopenshot][audiowaveformer][channels]" ) +{ + const std::string path = "AudioWaveformer-channels.wav"; + const int sample_rate = 48000; + WriteStereoWav(path, sample_rate, 3.0, [&](int channel, int64_t index) { + const double amplitude = channel == 0 ? 0.8 : 0.2; + return static_cast(amplitude * std::sin(2.0 * M_PI * 440.0 * index / sample_rate)); + }); + + FFmpegReader reader(path); + reader.Open(); + REQUIRE(reader.info.channels == 2); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 200; + + AudioWaveformData all = waveformer.ExtractChannelSamples(samples_per_second, false); + AudioWaveformData mixed = waveformer.ExtractSamples(-1, samples_per_second, false); + REQUIRE(all.channel_count() == 2); + REQUIRE(all.channel_rms_samples.size() == 2); + REQUIRE(all.max_samples.size() == mixed.max_samples.size()); + for (size_t i = 0; i < mixed.max_samples.size(); i++) { + CHECK(all.max_samples[i] == Approx(mixed.max_samples[i]).margin(1e-6)); + CHECK(all.rms_samples[i] == Approx(mixed.rms_samples[i]).margin(1e-6)); + } + + for (int channel = 0; channel < 2; channel++) { + AudioWaveformData single = waveformer.ExtractSamples(channel, samples_per_second, false); + const auto& channel_max = all.channel_max_samples[channel]; + const auto& channel_rms = all.channel_rms_samples[channel]; + REQUIRE(channel_max.size() == single.max_samples.size()); + REQUIRE(channel_rms.size() == single.rms_samples.size()); + for (size_t i = 0; i < single.max_samples.size(); i++) { + CHECK(channel_max[i] == Approx(single.max_samples[i]).margin(1e-6)); + CHECK(channel_rms[i] == Approx(single.rms_samples[i]).margin(1e-6)); + } + std::vector> vectors = all.channel_vectors(channel); + CHECK(vectors[0] == channel_max); + CHECK(vectors[1] == channel_rms); + } + + // Channels keep their own levels: left peaks near 0.8, right near 0.2, mix follows the loudest + const size_t middle = all.max_samples.size() / 2; + CHECK(all.channel_max_samples[0][middle] == Approx(0.8f).margin(0.01f)); + CHECK(all.channel_max_samples[1][middle] == Approx(0.2f).margin(0.01f)); + CHECK(all.max_samples[middle] == Approx(0.8f).margin(0.01f)); + CHECK(all.channel_rms_samples[0][middle] == Approx(0.8f / std::sqrt(2.0f)).margin(0.01f)); + CHECK(all.rms_samples[middle] == Approx(std::sqrt((0.32f + 0.02f) / 2.0f)).margin(0.01f)); + + // Out-of-range channel requests return empty envelopes + CHECK(all.channel_vectors(2)[0].empty()); + CHECK(all.channel_vectors(-1)[1].empty()); + + // Normalizing scales every envelope by the loudest mixed peak + AudioWaveformData normalized = waveformer.ExtractChannelSamples(samples_per_second, true); + const float scale = 1.0f / *std::max_element(all.max_samples.begin(), all.max_samples.end()); + CHECK(*std::max_element(normalized.max_samples.begin(), normalized.max_samples.end()) == Approx(1.0f).margin(1e-5)); + CHECK(normalized.channel_max_samples[1][middle] == Approx(all.channel_max_samples[1][middle] * scale).margin(1e-5)); + + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE( "Single-pass channel extraction of media without audio is empty", "[libopenshot][audiowaveformer][channels]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "front.png"; + FFmpegReader reader(path.str()); + AudioWaveformer waveformer(&reader); + AudioWaveformData waveform = waveformer.ExtractChannelSamples(200, false); + CHECK(waveform.max_samples.empty()); + CHECK(waveform.channel_count() == 0); + reader.Close(); +} + +TEST_CASE( "Single-pass channel extraction of a video file", "[libopenshot][audiowaveformer][channels]" ) +{ + std::stringstream path; + path << TEST_MEDIA_PATH << "sintel_trailer-720p.mp4"; + FFmpegReader reader(path.str()); + reader.Open(); + AudioWaveformer waveformer(&reader); + const int samples_per_second = 20; + AudioWaveformData waveform = waveformer.ExtractChannelSamples(samples_per_second, false); + const size_t expected_total = static_cast(std::ceil(reader.info.duration * samples_per_second)); + CHECK(waveform.max_samples.size() == expected_total); + REQUIRE(waveform.channel_count() == reader.info.channels); + for (int channel = 0; channel < waveform.channel_count(); channel++) { + CHECK(waveform.channel_max_samples[channel].size() == expected_total); + CHECK(waveform.channel_rms_samples[channel].size() == expected_total); + } + // Live reader keeps decoding video for preview + CHECK(reader.info.has_video); + reader.Close(); +} + +TEST_CASE( "Audio-only reader cache stays bounded on long sequential reads", "[libopenshot][ffmpegreader][audiowaveformer][cache]" ) +{ + // An audio-only cache budget sized from width x height never evicted, so every frame of + // the file stayed cached and each insert rescanned the whole cache (quadratic reads). + const std::string path = "AudioWaveformer-cache.wav"; + WriteStereoWav(path, 48000, 60.0, [](int, int64_t index) { + return static_cast(0.5 * std::sin(index * 0.01)); + }); + + FFmpegReader reader(path); + reader.Open(); + REQUIRE_FALSE(reader.info.has_video); + REQUIRE(reader.info.video_length > 1000); + for (int64_t frame = 1; frame <= reader.info.video_length; frame++) { + reader.GetFrame(frame); + } + CHECK(reader.GetCache()->Count() < 300); + CHECK(reader.GetCache()->Count() > 0); + reader.Close(); + std::remove(path.c_str()); +} + +TEST_CASE( "ApplyKeyframes maps time and volume curves onto envelopes", "[libopenshot][audiowaveformer][keyframes]" ) +{ + AudioWaveformer waveformer(nullptr); + const Fraction fps(30, 1); + const int samples_per_second = 30; // one envelope sample per frame keeps the math readable + std::vector ramp(60); + std::vector ramp_rms(60); + for (size_t i = 0; i < ramp.size(); i++) { + ramp[i] = static_cast(i) / 100.0f; + ramp_rms[i] = ramp[i] / 2.0f; + } + + SECTION("no keyframes keeps the envelope") { + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, nullptr, fps, samples_per_second); + CHECK(out.max_samples == ramp); + CHECK(out.rms_samples == ramp_rms); + } + + SECTION("identity time curve and unity volume keep the envelope") { + Keyframe time = LinearKeyframe({{1, 1}, {60, 60}}); + Keyframe volume(1.0); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, &volume, fps, samples_per_second); + REQUIRE(out.max_samples.size() == ramp.size()); + for (size_t i = 0; i < ramp.size(); i++) { + CHECK(out.max_samples[i] == Approx(ramp[i]).margin(1e-6)); + CHECK(out.rms_samples[i] == Approx(ramp_rms[i]).margin(1e-6)); + } + } + + SECTION("single-point time curve and empty volume curve are ignored") { + Keyframe time(1.0); + Keyframe volume; + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, &volume, fps, samples_per_second); + CHECK(out.max_samples == ramp); + } + + SECTION("volume curve scales max and RMS") { + Keyframe volume = LinearKeyframe({{1, 0.0}, {60, 1.0}}); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, &volume, fps, samples_per_second); + REQUIRE(out.max_samples.size() == ramp.size()); + CHECK(out.max_samples.front() == Approx(0.0f).margin(1e-6)); + CHECK(out.max_samples[30] == Approx(ramp[30] * volume.GetValue(31)).margin(1e-5)); + CHECK(out.rms_samples[30] == Approx(ramp_rms[30] * volume.GetValue(31)).margin(1e-5)); + CHECK(out.max_samples.back() == Approx(ramp.back()).margin(1e-5)); + } + + SECTION("reversed time curve reverses the envelope") { + Keyframe time = LinearKeyframe({{1, 60}, {60, 1}}); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, nullptr, fps, samples_per_second); + REQUIRE(out.max_samples.size() == ramp.size()); + CHECK(out.max_samples.front() == Approx(ramp.back()).margin(1e-6)); + CHECK(out.max_samples.back() == Approx(ramp.front()).margin(1e-6)); + } + + SECTION("slowed time curve doubles the length and interpolates") { + Keyframe time = LinearKeyframe({{1, 1}, {120, 60}}); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, ramp_rms, &time, nullptr, fps, samples_per_second); + REQUIRE(out.max_samples.size() == 120); + CHECK(out.max_samples.front() == Approx(ramp.front()).margin(1e-6)); + CHECK(out.max_samples[119] == Approx(ramp.back()).margin(1e-6)); + for (size_t i = 1; i < out.max_samples.size(); i++) { + CHECK(out.max_samples[i] >= out.max_samples[i - 1]); + } + } + + SECTION("short or empty RMS envelopes do not read out of bounds") { + std::vector short_rms(10, 0.5f); + AudioWaveformData out = waveformer.ApplyKeyframes(ramp, short_rms, nullptr, nullptr, fps, samples_per_second); + REQUIRE(out.rms_samples.size() == ramp.size()); + CHECK(out.rms_samples[5] == Approx(0.5f)); + CHECK(out.rms_samples[50] == Approx(0.0f)); + AudioWaveformData empty = waveformer.ApplyKeyframes(std::vector(), ramp_rms, nullptr, nullptr, fps, samples_per_second); + CHECK(empty.max_samples.empty()); + } + + SECTION("invalid density or fps returns empty data") { + CHECK(waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, nullptr, fps, 0).max_samples.empty()); + CHECK(waveformer.ApplyKeyframes(ramp, ramp_rms, nullptr, nullptr, Fraction(0, 1), samples_per_second).max_samples.empty()); + } +}