diff --git a/src/effects/Caption.cpp b/src/effects/Caption.cpp index 6e96092c0..c02f984e1 100644 --- a/src/effects/Caption.cpp +++ b/src/effects/Caption.cpp @@ -15,6 +15,14 @@ #include "../Clip.h" #include "../Timeline.h" +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "../VulkanCompositor.h" +#include "../VulkanFrameImage.h" +#include "../Logger.h" +#endif + +#include +#include #include #include #include @@ -26,6 +34,28 @@ using namespace openshot; +namespace { +// Fade the completed overlay, preserving text/stroke/background relationships. +void CompositeCaption(const std::shared_ptr& frame, const QImage& overlay, double opacity) { +#ifdef OPENSHOT_ENABLE_GPU_FRAME + if (VulkanRenderingEnabled() && + std::dynamic_pointer_cast(frame->GetDeferredImage())) { + try { + auto uploaded = UploadVulkanImage(overlay); + frame->AddDeferredImage(RenderVulkanImage(uploaded, frame->GetDeferredImage(), + frame->GetWidth(), frame->GetHeight(), QTransform(), opacity)); + return; + } catch (const std::exception& error) { + Logger::Instance()->Log(std::string("Vulkan caption CPU fallback: ") + error.what()); + } + } +#endif + QPainter painter(frame->GetImage().get()); + painter.setOpacity(opacity); + painter.drawImage(0, 0, overlay); +} +} + /// Blank constructor, useful when using Json to load the effect properties Caption::Caption() : color("#ffffff"), stroke("#a9a9a9"), background("#ff000000"), background_alpha(0.0), left(0.1), top(0.75), right(0.1), bottom(0.0), stroke_width(0.5), font_size(30.0), font_alpha(1.0), is_dirty(true), font_name("sans"), font(NULL), metrics(NULL), @@ -136,19 +166,11 @@ std::shared_ptr Caption::GetFrame(std::shared_ptrAddColor(image_size.width(), image_size.height(), "#000000"); } - // Get the frame's image - std::shared_ptr frame_image = frame->GetImage(); + const QSize frame_size(frame->GetWidth(), frame->GetHeight()); // Calculate scale factor, to keep different resolutions from // having dramatically different font sizes - double timeline_scale_factor = frame_image->width() / 600.0; - - // Load timeline's new frame image into a QPainter - QPainter painter(frame_image.get()); - painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true); - - // Composite a new layer onto the image - painter.setCompositionMode(QPainter::CompositionMode_SourceOver); + double timeline_scale_factor = frame_size.width() / 600.0; // Font options and metrics for caption text double font_size_value = font_size.GetValue(frame_number) * timeline_scale_factor; @@ -170,16 +192,16 @@ std::shared_ptr Caption::GetFrame(std::shared_ptrwidth() * left_value; - double top_margin_y = frame_image->height() * top_value; + double left_margin_x = frame_size.width() * left_value; + double top_margin_y = frame_size.height() * top_value; double starting_y = top_margin_y + metrics_line_spacing; double current_y = starting_y; double bottom_y = starting_y; double top_y = starting_y; double max_text_width = 0.0; - double right_margin_x = frame_image->width() - (frame_image->width() * right_value); + double right_margin_x = frame_size.width() - (frame_size.width() * right_value); double caption_area_width = right_margin_x - left_margin_x; - double bottom_margin_y = frame_image->height() - (frame_image->height() * bottom_value); + double bottom_margin_y = frame_size.height() - (frame_size.height() * bottom_value); double caption_area_height = std::max(bottom_margin_y - starting_y, 0.0); QRectF caption_area = QRectF(left_margin_x, starting_y, caption_area_width, caption_area_height); QRectF caption_clip_area = QRectF(left_margin_x, top_margin_y, caption_area_width, @@ -300,6 +322,11 @@ std::shared_ptr Caption::GetFrame(std::shared_ptr Caption::GetFrame(std::shared_ptr= 0.0 && fade_out_percentage <= 1.0 ? fade_out_percentage : 1.0); + if (cue_alpha <= 0.0) + return frame; + + // Cache the evaluated drawing commands, excluding fade opacity. Both CPU + // and Vulkan reuse the full-opacity raster throughout a cue's fade. + QByteArray overlay_key; + QDataStream key(&overlay_key, QIODevice::WriteOnly); + key << frame_size << caption_clip_area << caption_area_with_padding + << alignment_offset << background_corner_value << stroke_width_value + << QString::fromStdString(background.GetColorHex(frame_number)) + << QString::fromStdString(color.GetColorHex(frame_number)) + << QString::fromStdString(stroke.GetColorHex(frame_number)) + << background_alpha.GetValue(frame_number) << font_alpha.GetValue(frame_number); + for (const auto& path : text_paths) + key << path; + if (overlay_key == this->overlay_key) { + CompositeCaption(frame, overlay_cache, cue_alpha); + return frame; + } + QImage overlay(frame_size, QImage::Format_RGBA8888_Premultiplied); + overlay.fill(Qt::transparent); + + // Paint the caption independently of the video. + QPainter painter(&overlay); + painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true); + + // Composite a new layer onto the image + painter.setCompositionMode(QPainter::CompositionMode_SourceOver); + // Set background color of caption QBrush background_brush; QColor background_qcolor = QColor(QString(background.GetColorHex(frame_number).c_str())); // Align background center caption_area_with_padding.translate(alignment_offset, 0.0); - if (fade_in_percentage < 1.0) { - // Fade in background - background_qcolor.setAlphaF(fade_in_percentage * background_alpha.GetValue(frame_number)); - } else if (fade_out_percentage >= 0.0 && fade_out_percentage <= 1.0) { - // Fade out background - background_qcolor.setAlphaF(fade_out_percentage * background_alpha.GetValue(frame_number)); - } else { - background_qcolor.setAlphaF(background_alpha.GetValue(frame_number)); - } + background_qcolor.setAlphaF(background_alpha.GetValue(frame_number)); background_brush.setColor(background_qcolor); background_brush.setStyle(Qt::SolidPattern); painter.setBrush(background_brush); @@ -350,15 +400,6 @@ std::shared_ptr Caption::GetFrame(std::shared_ptr= 0.0 && fade_out_percentage <= 1.0) { - // Fade out text - font_qcolor.setAlphaF(fade_out_percentage * font_alpha.GetValue(frame_number)); - stroke_qcolor.setAlphaF(fade_out_percentage * font_alpha.GetValue(frame_number)); - } pen.setColor(stroke_qcolor); font_brush.setColor(font_qcolor); @@ -385,6 +426,10 @@ std::shared_ptr Caption::GetFrame(std::shared_ptroverlay_key = overlay_key; + overlay_cache = std::move(overlay); + CompositeCaption(frame, overlay_cache, cue_alpha); + // return the modified frame return frame; } diff --git a/src/effects/Caption.h b/src/effects/Caption.h index 45f9cdac9..51368505d 100644 --- a/src/effects/Caption.h +++ b/src/effects/Caption.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,10 @@ class Caption : public EffectBase QFontMetrics* metrics; ///< Font metrics object QFont* font; ///< QFont object bool is_dirty; + QByteArray overlay_key; ///< Evaluated drawing commands for the last caption. + QImage overlay_cache; ///< Last caption raster; stable cache key reuses the shared Vulkan upload. + + bool RendersOnVulkan() const override { return true; } /// Init effect settings void init_effect_details(); diff --git a/tests/Caption.cpp b/tests/Caption.cpp index d848c35a1..cee9abfd1 100644 --- a/tests/Caption.cpp +++ b/tests/Caption.cpp @@ -16,6 +16,8 @@ #include "openshot_catch.h" #include #include +#include +#include #include "effects/Caption.h" #include "Clip.h" #include "Frame.h" @@ -291,3 +293,259 @@ TEST_CASE("caption effect", "[libopenshot][caption]") { // Close QApplication app.quit(); } + +TEST_CASE("Caption fades composite the complete overlay on the CPU", "[caption]") { + int argc = 1; + char name[] = "caption-fade-test"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + openshot::Timeline timeline(600, 300, openshot::Fraction(24, 1), 44100, 2, openshot::LAYOUT_STEREO); + openshot::Caption caption("00:00.000 --> 00:02.000\nComplete caption"); + timeline.AddEffect(&caption); + caption.top = openshot::Keyframe(0.2); + caption.background_alpha = openshot::Keyframe(0.8); + caption.background = openshot::Color("#204080"); + caption.stroke_width = openshot::Keyframe(4.0); + auto full = std::make_shared(13, 600, 300, "#00000000"); + auto transparent = std::make_shared(600, 300, QImage::Format_RGBA8888_Premultiplied); + transparent->fill(Qt::transparent); + full->AddImage(transparent); + caption.GetFrame(full, 13); + const QImage overlay = full->GetImage()->copy(); + caption.fade_in = openshot::Keyframe(1.0); + caption.fade_out = openshot::Keyframe(1.0); + const std::vector> samples = { + {1, 0.0}, {7, 0.25}, {13, 0.5}, {19, 0.75}, {25, 23.0 / 24.0}, + {36, 0.5}, {48, 0.0}, {49, 0.0}, {13, 0.5}}; + for (const auto& sample : samples) { + INFO("Frame " << sample.first); + QImage expected(600, 300, QImage::Format_RGBA8888_Premultiplied); + expected.fill(QColor(90, 30, 60, 210)); + auto frame = std::make_shared(sample.first, 600, 300, "#00000000"); + frame->AddImage(std::make_shared(expected)); + QPainter painter(&expected); + painter.setOpacity(sample.second); + painter.drawImage(0, 0, overlay); + painter.end(); + caption.GetFrame(frame, sample.first); + const auto actual = frame->GetImage()->convertToFormat(QImage::Format_RGBA8888_Premultiplied); + int largest_error = 0; + for (int y = 0; y < actual.height(); ++y) + for (int x = 0; x < actual.width() * 4; ++x) + largest_error = std::max(largest_error, std::abs(int(actual.constScanLine(y)[x]) - int(expected.constScanLine(y)[x]))); + CHECK(largest_error <= 2); + } + timeline.RemoveEffect(&caption); +} + +#ifdef OPENSHOT_ENABLE_GPU_FRAME +#include "VulkanFrameImage.h" +#include "GpuMetrics.h" +#include "GpuDevices.h" +#include "FFmpegReader.h" +#include "FFmpegWriter.h" +#include "QtImageReader.h" +#include +#include + +namespace { +int CaptionPixelError(const std::shared_ptr& cpu, + const std::shared_ptr& gpu) { + const auto expected = cpu->GetImage()->convertToFormat(QImage::Format_RGBA8888_Premultiplied); + const auto actual = gpu->GetImage()->convertToFormat(QImage::Format_RGBA8888_Premultiplied); + int largest_error = 0; + for (int y = 0; y < actual.height(); ++y) + for (int x = 0; x < actual.width() * 4; ++x) + largest_error = std::max(largest_error, std::abs(int(actual.constScanLine(y)[x]) - int(expected.constScanLine(y)[x]))); + return largest_error; +} +} + +TEST_CASE("Vulkan captions preserve video surfaces and match CPU rendering", "[caption][vulkan]") { + if (!openshot::VulkanRenderingEnabled() || openshot::ActiveGpuDevice().name.empty()) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return; + } + int argc = 1; + char name[] = "caption-test"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + openshot::Timeline timeline(600, 300, openshot::Fraction(24, 1), 44100, 2, openshot::LAYOUT_STEREO); + openshot::Caption caption("00:01.000 --> 00:03.000\nFirst caption\nSecond line\n\n" + "00:03.500 --> 00:04.000\nNext cue"); + timeline.AddEffect(&caption); + caption.top = openshot::Keyframe(0.2); + caption.bottom = openshot::Keyframe(0.15); + caption.background_alpha = openshot::Keyframe(0.6); + caption.font_alpha = openshot::Keyframe(0.7); + caption.stroke_width = openshot::Keyframe(2.0); + caption.fade_in = openshot::Keyframe(0.5); + caption.fade_out = openshot::Keyframe(0.5); + caption.font_size.AddPoint(40, 30.0, openshot::LINEAR); + caption.font_size.AddPoint(60, 45.0, openshot::LINEAR); + + // A partially transparent source also checks premultiplied source-over. + QImage input(600, 300, QImage::Format_RGBA8888_Premultiplied); + input.fill(QColor(25, 65, 110, 180)); + int iteration = 0; + for (int n : {1, 25, 30, 40, 40, 41, 40, 65, 72, 73, 85, 90, 96, 97}) { + // Change the video under repeated cues to detect accidental video caching. + input.fill(iteration++ % 2 ? QColor(25, 65, 110, 180) : QColor(90, 30, 60, 210)); + auto surface = openshot::UploadVulkanImage(input); + INFO("Frame " << n); + auto cpu = std::make_shared(n, 600, 300, "#00000000"); + cpu->AddImage(std::make_shared(input)); + caption.GetFrame(cpu, n); + auto gpu = std::make_shared(n, 600, 300, "#00000000"); + gpu->AddDeferredImage(surface); + const auto before = openshot::VulkanFrameReadbackCount(); + caption.ProcessFrame(gpu, n); + REQUIRE(gpu->GetDeferredImage()); + CHECK(openshot::VulkanFrameReadbackCount() == before); + if (n == 1 || n == 73 || n == 97) + CHECK(gpu->GetDeferredImage() == surface); + CHECK(CaptionPixelError(cpu, gpu) <= 3); + } + + // Editing text/style and changing resolution must invalidate a static cue. + caption.fade_in = openshot::Keyframe(0.0); + caption.fade_out = openshot::Keyframe(0.0); + QTemporaryDir directory; + REQUIRE(directory.isValid()); + QImage mask(600, 300, QImage::Format_RGB888); + mask.fill(QColor(128, 128, 128)); + const auto mask_path = directory.filePath("mask.png").toStdString(); + REQUIRE(mask.save(QString::fromStdString(mask_path))); + for (int pass = 0; pass < 6; ++pass) { + INFO("Edit pass " << pass); + if (pass == 1) caption.CaptionText("00:00.000 --> 00:10.000\nChanged caption"); + if (pass == 2) caption.color = openshot::Color("#ff0000"); + if (pass == 4) caption.MaskReader(new openshot::QtImageReader(mask_path)); + if (pass == 5) caption.mask_invert = true; + const int width = pass == 3 ? 300 : 600; + QImage pixels(width, 300, QImage::Format_RGBA8888_Premultiplied); + pixels.fill(Qt::black); + auto cpu = std::make_shared(40, width, 300, "#000000"); + cpu->AddImage(std::make_shared(pixels)); + caption.ProcessFrame(cpu, 40); + auto gpu = std::make_shared(40, width, 300, "#000000"); + gpu->AddDeferredImage(openshot::UploadVulkanImage(pixels)); + const auto before = openshot::VulkanFrameReadbackCount(); + caption.ProcessFrame(gpu, 40); + REQUIRE(gpu->GetDeferredImage()); + CHECK(openshot::VulkanFrameReadbackCount() == before); + CHECK(CaptionPixelError(cpu, gpu) <= 3); + } + timeline.RemoveEffect(&caption); +} + +TEST_CASE("Vulkan Caption fades reuse one uploaded overlay", "[caption][vulkan]") { + if (!openshot::VulkanRenderingEnabled() || openshot::ActiveGpuDevice().name.empty()) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return; + } + int argc = 1; + char name[] = "caption-upload-test"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + openshot::Timeline timeline(600, 300, openshot::Fraction(24, 1), 44100, 2, openshot::LAYOUT_STEREO); + openshot::Caption caption("00:00.000 --> 00:02.000\nOne uploaded caption"); + timeline.AddEffect(&caption); + caption.top = openshot::Keyframe(0.2); + caption.background_alpha = openshot::Keyframe(0.8); + caption.fade_in = openshot::Keyframe(0.5); + caption.fade_out = openshot::Keyframe(0.5); + QImage input(600, 300, QImage::Format_RGBA8888_Premultiplied); + input.fill(Qt::black); + auto surface = openshot::UploadVulkanImage(input); + openshot::ResetGpuMetrics(); + const auto readbacks = openshot::VulkanFrameReadbackCount(); + for (int n = 1; n <= 49; ++n) { + openshot::gpu_metrics::Scope scope(false); + auto frame = std::make_shared(n, 600, 300, "#000000"); + frame->AddDeferredImage(surface); + caption.GetFrame(frame, n); + REQUIRE(frame->GetDeferredImage()); + } + auto counts = openshot::stringToJson(openshot::GpuMetricsSnapshot()); + CHECK(counts["uploads"].asUInt64() == 1); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + // A style edit must still produce a new raster/upload. + caption.color = openshot::Color("#ff0000"); + { + openshot::gpu_metrics::Scope scope(false); + auto frame = std::make_shared(13, 600, 300, "#000000"); + frame->AddDeferredImage(surface); + caption.GetFrame(frame, 13); + } + counts = openshot::stringToJson(openshot::GpuMetricsSnapshot()); + CHECK(counts["uploads"].asUInt64() == 2); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + timeline.RemoveEffect(&caption); +} + +TEST_CASE("Vulkan H264 decode caption encode requires no video readbacks", "[caption][vulkan][hardware][writer]") { + if (!openshot::VulkanRenderingEnabled() || openshot::ActiveGpuDevice().name.empty()) { + WARN("Skipping Vulkan test: no Vulkan device is available"); + return; + } + const auto encoders = openshot::GpuEncoders(); + const auto encoder = std::find(encoders.begin(), encoders.end(), "h264_vulkan") != encoders.end() + ? "h264_vulkan" : "h264_nvenc"; + if (std::find(encoders.begin(), encoders.end(), encoder) == encoders.end()) { + WARN("Skipping Vulkan test: no GPU H.264 encoder"); + return; + } + int argc = 1; + char name[] = "caption-export-test"; + char* argv[] = {name, nullptr}; + QApplication app(argc, argv); + QTemporaryDir directory; + REQUIRE(directory.isValid()); + openshot::FFmpegReader reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + openshot::Caption caption("00:00.000 --> 00:00.500\nVulkan caption\nSecond line"); + caption.top = openshot::Keyframe(0.25); + caption.background_alpha = openshot::Keyframe(0.7); + caption.fade_in = openshot::Keyframe(0.2); + caption.fade_out = openshot::Keyframe(0.2); + openshot::Clip clip(&reader); + clip.AddEffect(&caption); + openshot::Timeline timeline(640, 360, openshot::Fraction(24, 1), 48000, 2, openshot::LAYOUT_STEREO); + timeline.SetMaxSize(640, 360); + timeline.AddClip(&clip); + timeline.Open(); + const auto path = directory.filePath("caption.mp4").toStdString(); + openshot::FFmpegWriter writer(path); + writer.SetVideoOptions(true, encoder, openshot::Fraction(24, 1), 640, 360, + openshot::Fraction(1, 1), false, false, 2000000); + writer.Open(); + REQUIRE(writer.info.vcodec == encoder); + std::shared_ptr reference; + const auto before = openshot::VulkanFrameReadbackCount(); + for (int n = 1; n <= 24; ++n) { + auto frame = timeline.GetFrame(n); + REQUIRE(frame->GetDeferredImage()); + writer.WriteFrame(frame); + REQUIRE(frame->GetDeferredImage()); + if (n == 8) reference = frame; + } + writer.Close(); + CHECK(reader.HardwareDecodeSuccessful()); + CHECK(openshot::VulkanFrameReadbackCount() == before); + timeline.Close(); + openshot::FFmpegReader encoded(path); + encoded.Open(); + CHECK(encoded.info.video_length == 24); + CHECK(encoded.info.width == 640); + CHECK(encoded.info.height == 360); + const auto expected = reference->GetImage()->convertToFormat(QImage::Format_RGB888); + const auto actual = encoded.GetFrame(8)->GetImage()->convertToFormat(QImage::Format_RGB888); + double error = 0.0; + for (int y = 0; y < actual.height(); ++y) + for (int x = 0; x < actual.width() * 3; ++x) + error += std::abs(int(actual.constScanLine(y)[x]) - int(expected.constScanLine(y)[x])); + CHECK(error / (actual.width() * actual.height() * 3) < 10.0); + encoded.Close(); + clip.RemoveEffect(&caption); +} +#endif diff --git a/tests/VulkanPresentation.cpp b/tests/VulkanPresentation.cpp index 359ca948e..e30ec59bb 100644 --- a/tests/VulkanPresentation.cpp +++ b/tests/VulkanPresentation.cpp @@ -2,6 +2,10 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "openshot_catch.h" #include "Frame.h" +#include "Clip.h" +#include "Timeline.h" +#include "FFmpegReader.h" +#include "effects/Caption.h" #include "VulkanFrameImage.h" #include "VulkanCompositor.h" #include "GpuDevices.h" @@ -229,6 +233,32 @@ TEST_CASE("Vulkan presentation keeps image on GPU and preserves Qt overlays", "[ CHECK(widget.presenter.FallbackFrames() == 0); CHECK(image.pixelColor(image.width()/2, image.height()/4) == QColor(Qt::blue)); CHECK(image.pixelColor(image.width()/2, 3*image.height()/4) == QColor(Qt::red)); + // H.264 decode -> Caption -> timeline transform -> native presentation. + // Cue changes, repeated cues, and empty frames must all retain GPU pixels. + openshot::FFmpegReader reader(TEST_MEDIA_PATH "sintel_trailer-720p.mp4"); + openshot::Caption caption("00:00.000 --> 00:00.500\nVulkan caption"); + caption.top = openshot::Keyframe(0.3); + openshot::Clip clip(&reader); + clip.AddEffect(&caption); + openshot::Timeline timeline(640, 360, openshot::Fraction(24, 1), 48000, 2, openshot::LAYOUT_STEREO); + timeline.SetMaxSize(640, 360); + timeline.AddClip(&clip); + timeline.Open(); + for (int n : {2, 8, 9, 24}) { + auto captioned = timeline.GetFrame(n); + REQUIRE(captioned->GetDeferredImage()); + renderer.paint(captioned); + QCoreApplication::processEvents(); + image = widget.grabFramebuffer(); + CHECK(widget.nativeDraw); + CHECK(widget.presenter.FallbackFrames() == 0); + CHECK(openshot::VulkanFrameReadbackCount() == readbacks); + REQUIRE(captioned->GetDeferredImage()); + } + CHECK(reader.HardwareDecodeSuccessful()); + timeline.Close(); + clip.RemoveEffect(&caption); + // Recovery must not read pixels from a dead device or throw through paintGL. openshot::ReportVulkanResult(VK_ERROR_DEVICE_LOST); CHECK_NOTHROW(widget.presenter.Present(frame));