feat: Add frame interceptors to FrameOutput and Preview

packages/react-native-vision-camera/VisionCamera.podspec

@@ -23,6 +23,15 @@ Pod::Spec.new do |s|
   ]
   s.frameworks = ["AVFoundation"]
 
+  # Frame interceptors (ios/Public/FrameInterceptors.swift) are compiled in only for the configurations
+  # the app lists in its package.json under "visionCamera.frameInterceptors.ios" (default: Debug).
+  app_package_json = File.join(Pod::Config.instance.installation_root.to_s, "..", "package.json")
+  app_package = File.exist?(app_package_json) ? JSON.parse(File.read(app_package_json)) : {}
+  frame_interceptor_configurations = app_package.dig("visionCamera", "frameInterceptors", "ios") || ["Debug"]
+  s.pod_target_xcconfig = frame_interceptor_configurations.to_h do |configuration|
+    ["SWIFT_ACTIVE_COMPILATION_CONDITIONS[config=#{configuration}]", "$(inherited) VISION_CAMERA_FRAME_INTERCEPTORS"]
+  end
+
   load 'nitrogen/generated/ios/VisionCamera+autolinking.rb'
   add_nitrogen_files(s)
 

packages/react-native-vision-camera/android/build.gradle

@@ -18,6 +18,14 @@ def isNewArchitectureEnabled() {
   return rootProject.hasProperty("newArchEnabled") && rootProject.getProperty("newArchEnabled") == "true"
 }
 
+// Build types that compile frame interceptors (FrameInterceptors.kt) in, as listed in the app's
+// package.json under "visionCamera.frameInterceptors.android" (default: debug).
+def frameInterceptorBuildTypes() {
+  def appPackageJson = new File(rootProject.projectDir, "../package.json")
+  def appPackage = appPackageJson.exists() ? new groovy.json.JsonSlurper().parse(appPackageJson) : [:]
+  return appPackage.visionCamera?.frameInterceptors?.android ?: ["debug"]
+}
+
 apply plugin: "com.android.library"
 apply plugin: 'org.jetbrains.kotlin.android'
 apply from: '../nitrogen/generated/android/VisionCamera+autolinking.gradle'
@@ -110,6 +118,19 @@ android {
     }
   }
 
+  def interceptorBuildTypes = frameInterceptorBuildTypes()
+  // A custom build type of the app (e.g. "qa") must exist here too, otherwise the app would get
+  // this library's release variant, which has frame interceptors compiled out.
+  interceptorBuildTypes.findAll { name -> buildTypes.findByName(name) == null }.each { name ->
+    buildTypes.create(name) {
+      initWith(buildTypes.release)
+      matchingFallbacks = ["release"]
+    }
+  }
+  buildTypes.configureEach { buildType ->
+    buildType.buildConfigField "boolean", "FRAME_INTERCEPTORS", String.valueOf(buildType.name in interceptorBuildTypes)
+  }
+
   lintOptions {
     disable "GradleCompatible"
   }

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt

@@ -19,6 +19,7 @@ import com.margelo.nitro.camera.extensions.sortedByClosestTo
 import com.margelo.nitro.camera.extensions.surfaceRotation
 import com.margelo.nitro.camera.hybrids.HybridNativeThread
 import com.margelo.nitro.camera.hybrids.instances.HybridFrame
+import com.margelo.nitro.camera.public.FrameInterceptors
 import com.margelo.nitro.camera.public.NativeCameraOutput
 import com.margelo.nitro.camera.utils.IdentifiableExecutor
 
@@ -114,6 +115,7 @@ class HybridFrameOutput(
 
     if (onFrame != null) {
       imageAnalysis.setAnalyzer(executor) { image ->
+        val image = FrameInterceptors.intercept(image, mirrorMode == MirrorMode.ON) ?: return@setAnalyzer
         // This represents the Image's orientation relative to the
         // Frame Output. If `enablePhysicalBufferRotation` is true,
         // it will always be `UP` - otherwise it will be whatever

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/public/FrameInterceptors.kt

@@ -0,0 +1,78 @@
+package com.margelo.nitro.camera.public
+
+import android.util.Log
+import androidx.camera.core.ImageProxy
+import androidx.camera.view.PreviewView
+import com.margelo.nitro.camera.BuildConfig
+
+/**
+ * Optional extension point that decorates FrameOutput frame delivery and PreviewView creation
+ * (e.g. to inject test images). Interceptors are only accepted in build types listed in the app's
+ * package.json under "visionCamera.frameInterceptors.android" (default: debug). While nothing is
+ * registered, FrameOutput and PreviewView behave as usual.
+ */
+interface NativeFrameInterceptor {
+  /**
+   * Called on a FrameOutput's executor for every camera frame. Returns the frame to deliver: [image],
+   * a replacement, or null to drop it. If [image] is not returned, the interceptor must close it.
+   */
+  fun intercept(
+    image: ImageProxy,
+    isMirrored: Boolean,
+  ): ImageProxy?
+
+  /** Called on the UI thread when a PreviewView is created. */
+  fun onPreviewViewCreated(view: PreviewView)
+}
+
+object FrameInterceptors {
+  private const val TAG = "FrameInterceptors"
+
+  /** Whether this build of VisionCamera accepts an interceptor (decided at compile time). */
+  val isEnabled: Boolean
+    get() = BuildConfig.FRAME_INTERCEPTORS
+
+  @Volatile
+  private var registered: NativeFrameInterceptor? = null
+
+  /**
+   * Registers [interceptor]. Only one interceptor can be registered, and only if [isEnabled];
+   * otherwise the call is refused, logged and returns `false`.
+   */
+  @Synchronized
+  fun register(interceptor: NativeFrameInterceptor): Boolean {
+    val name = interceptor.javaClass.name
+    if (!isEnabled) {
+      Log.e(TAG, "Frame interceptors are disabled in this build; ignoring $name.")
+      return false
+    }
+    registered?.let {
+      Log.e(TAG, "Frame interceptor ${it.javaClass.name} is already registered; ignoring $name.")
+      return false
+    }
+    registered = interceptor
+    Log.w(TAG, "Frame interceptor $name registered: camera frames and previews can be replaced in this build.")
+    return true
+  }
+
+  internal fun intercept(
+    image: ImageProxy,
+    isMirrored: Boolean,
+  ): ImageProxy? {
+    val interceptor = registered ?: return image
+    return try {
+      interceptor.intercept(image, isMirrored)
+    } catch (e: Throwable) {
+      Log.e(TAG, "Frame interceptor failed; delivering the camera frame.", e)
+      image
+    }
+  }
+
+  internal fun previewCreated(view: PreviewView) {
+    try {
+      registered?.onPreviewViewCreated(view)
+    } catch (e: Throwable) {
+      Log.e(TAG, "Frame interceptor failed to decorate a PreviewView.", e)
+    }
+  }
+}

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/views/HybridPreviewView.kt

@@ -23,6 +23,7 @@ import com.margelo.nitro.camera.extensions.converters.toImplementationMode
 import com.margelo.nitro.camera.extensions.converters.toScaleType
 import com.margelo.nitro.camera.extensions.installHierarchyFitter
 import com.margelo.nitro.camera.hybrids.metering.HybridMeteringPoint
+import com.margelo.nitro.camera.public.FrameInterceptors
 import com.margelo.nitro.camera.public.NativeGestureController
 import com.margelo.nitro.camera.public.NativePreviewOutput
 import com.margelo.nitro.core.Promise
@@ -98,6 +99,7 @@ class HybridPreviewView(
     previewView.setOnTouchListener(this)
     meteringPointFactory = previewView.meteringPointFactory
     pixelRatio = context.resources.displayMetrics.density
+    FrameInterceptors.previewCreated(previewView)
   }
 
   override fun dispose() {

packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift

@@ -157,6 +157,7 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu
           at: timestamp,
           orientation: bufferOrientation,
           isMirrored: isBufferMirrored)
+        guard let sampleBuffer = FrameInterceptors.intercept(sampleBuffer, orientation: metadata.orientation, isMirrored: metadata.isMirrored) else { return }
         let frame = HybridFrame(
           buffer: sampleBuffer,
           metadata: metadata)

packages/react-native-vision-camera/ios/Hybrid Objects/Views/HybridPreviewView.swift

@@ -133,6 +133,7 @@ final class HybridPreviewView: HybridPreviewViewSpec {
           self.view.layer.addSublayer(previewLayer)
         }
         previewLayer.frame = self.view.bounds
+        FrameInterceptors.previewAttached(self.view)
 
         // Update resizeMode
         let resizeMode = self.resizeMode ?? .cover

packages/react-native-vision-camera/ios/Public/FrameInterceptors.swift

@@ -0,0 +1,75 @@
+///
+/// FrameInterceptors.swift
+/// VisionCamera
+///
+/// Optional extension point that decorates FrameOutput frame delivery and PreviewView attachment
+/// (e.g. to inject test images). Interceptors are only accepted in builds compiled with
+/// VISION_CAMERA_FRAME_INTERCEPTORS, set for the configurations listed in the app's package.json
+/// under "visionCamera.frameInterceptors.ios" (default: Debug). While nothing is registered,
+/// FrameOutput and PreviewView behave as usual.
+///
+
+import AVFoundation
+import Foundation
+import UIKit
+
+public protocol NativeFrameInterceptor: AnyObject {
+  /// Called on a FrameOutput's queue for every camera frame. Returns the frame to deliver: the
+  /// camera's `sampleBuffer`, a replacement, or nil to drop it.
+  func intercept(
+    _ sampleBuffer: CMSampleBuffer, orientation: CameraOrientation, isMirrored: Bool
+  ) -> CMSampleBuffer?
+  /// Called on the main thread after a PreviewView attached its preview layer to `view`.
+  func previewViewDidAttach(_ view: UIView)
+}
+
+public enum FrameInterceptors {
+  /// Whether this build of VisionCamera accepts an interceptor (decided at compile time).
+  public static var isEnabled: Bool {
+    #if VISION_CAMERA_FRAME_INTERCEPTORS
+      return true
+    #else
+      return false
+    #endif
+  }
+
+  private static let lock = NSLock()
+  private static var registered: (any NativeFrameInterceptor)?
+
+  /// Registers `interceptor`. Only one interceptor can be registered, and only if `isEnabled`;
+  /// otherwise the call is refused, logged and returns `false`.
+  @discardableResult
+  public static func register(_ interceptor: any NativeFrameInterceptor) -> Bool {
+    let name = String(describing: type(of: interceptor))
+    guard isEnabled else {
+      logger.error("Frame interceptors are disabled in this build; ignoring \(name).")
+      return false
+    }
+    lock.lock()
+    defer { lock.unlock() }
+    if let current = registered {
+      logger.error("Frame interceptor \(String(describing: type(of: current))) is already registered; ignoring \(name).")
+      return false
+    }
+    registered = interceptor
+    logger.warning("Frame interceptor \(name) registered: camera frames and previews can be replaced in this build.")
+    return true
+  }
+
+  private static var current: (any NativeFrameInterceptor)? {
+    lock.lock()
+    defer { lock.unlock() }
+    return registered
+  }
+
+  static func intercept(
+    _ sampleBuffer: CMSampleBuffer, orientation: CameraOrientation, isMirrored: Bool
+  ) -> CMSampleBuffer? {
+    guard let interceptor = current else { return sampleBuffer }
+    return interceptor.intercept(sampleBuffer, orientation: orientation, isMirrored: isMirrored)
+  }
+
+  static func previewAttached(_ view: UIView) {
+    current?.previewViewDidAttach(view)
+  }
+}