diff --git a/packages/react-native-vision-camera/VisionCamera.podspec b/packages/react-native-vision-camera/VisionCamera.podspec index ae1e07bd2b..03e6be8627 100644 --- a/packages/react-native-vision-camera/VisionCamera.podspec +++ b/packages/react-native-vision-camera/VisionCamera.podspec @@ -23,6 +23,15 @@ Pod::Spec.new do |s| ] s.frameworks = ["AVFoundation"] + # Frame interceptors (ios/Public/FrameInterceptors.swift) are compiled in only for the configurations + # the app lists in its package.json under "visionCamera.frameInterceptors.ios" (default: Debug). + app_package_json = File.join(Pod::Config.instance.installation_root.to_s, "..", "package.json") + app_package = File.exist?(app_package_json) ? JSON.parse(File.read(app_package_json)) : {} + frame_interceptor_configurations = app_package.dig("visionCamera", "frameInterceptors", "ios") || ["Debug"] + s.pod_target_xcconfig = frame_interceptor_configurations.to_h do |configuration| + ["SWIFT_ACTIVE_COMPILATION_CONDITIONS[config=#{configuration}]", "$(inherited) VISION_CAMERA_FRAME_INTERCEPTORS"] + end + load 'nitrogen/generated/ios/VisionCamera+autolinking.rb' add_nitrogen_files(s) diff --git a/packages/react-native-vision-camera/android/build.gradle b/packages/react-native-vision-camera/android/build.gradle index bfcb0e018f..02d2619fad 100644 --- a/packages/react-native-vision-camera/android/build.gradle +++ b/packages/react-native-vision-camera/android/build.gradle @@ -18,6 +18,14 @@ def isNewArchitectureEnabled() { return rootProject.hasProperty("newArchEnabled") && rootProject.getProperty("newArchEnabled") == "true" } +// Build types that compile frame interceptors (FrameInterceptors.kt) in, as listed in the app's +// package.json under "visionCamera.frameInterceptors.android" (default: debug). +def frameInterceptorBuildTypes() { + def appPackageJson = new File(rootProject.projectDir, "../package.json") + def appPackage = appPackageJson.exists() ? new groovy.json.JsonSlurper().parse(appPackageJson) : [:] + return appPackage.visionCamera?.frameInterceptors?.android ?: ["debug"] +} + apply plugin: "com.android.library" apply plugin: 'org.jetbrains.kotlin.android' apply from: '../nitrogen/generated/android/VisionCamera+autolinking.gradle' @@ -110,6 +118,19 @@ android { } } + def interceptorBuildTypes = frameInterceptorBuildTypes() + // A custom build type of the app (e.g. "qa") must exist here too, otherwise the app would get + // this library's release variant, which has frame interceptors compiled out. + interceptorBuildTypes.findAll { name -> buildTypes.findByName(name) == null }.each { name -> + buildTypes.create(name) { + initWith(buildTypes.release) + matchingFallbacks = ["release"] + } + } + buildTypes.configureEach { buildType -> + buildType.buildConfigField "boolean", "FRAME_INTERCEPTORS", String.valueOf(buildType.name in interceptorBuildTypes) + } + lintOptions { disable "GradleCompatible" } diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt index 41ddfad34b..7486eec13c 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt @@ -19,6 +19,7 @@ import com.margelo.nitro.camera.extensions.sortedByClosestTo import com.margelo.nitro.camera.extensions.surfaceRotation import com.margelo.nitro.camera.hybrids.HybridNativeThread import com.margelo.nitro.camera.hybrids.instances.HybridFrame +import com.margelo.nitro.camera.public.FrameInterceptors import com.margelo.nitro.camera.public.NativeCameraOutput import com.margelo.nitro.camera.utils.IdentifiableExecutor @@ -114,6 +115,7 @@ class HybridFrameOutput( if (onFrame != null) { imageAnalysis.setAnalyzer(executor) { image -> + val image = FrameInterceptors.intercept(image, mirrorMode == MirrorMode.ON) ?: return@setAnalyzer // This represents the Image's orientation relative to the // Frame Output. If `enablePhysicalBufferRotation` is true, // it will always be `UP` - otherwise it will be whatever diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/public/FrameInterceptors.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/public/FrameInterceptors.kt new file mode 100644 index 0000000000..b6c4318970 --- /dev/null +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/public/FrameInterceptors.kt @@ -0,0 +1,78 @@ +package com.margelo.nitro.camera.public + +import android.util.Log +import androidx.camera.core.ImageProxy +import androidx.camera.view.PreviewView +import com.margelo.nitro.camera.BuildConfig + +/** + * Optional extension point that decorates FrameOutput frame delivery and PreviewView creation + * (e.g. to inject test images). Interceptors are only accepted in build types listed in the app's + * package.json under "visionCamera.frameInterceptors.android" (default: debug). While nothing is + * registered, FrameOutput and PreviewView behave as usual. + */ +interface NativeFrameInterceptor { + /** + * Called on a FrameOutput's executor for every camera frame. Returns the frame to deliver: [image], + * a replacement, or null to drop it. If [image] is not returned, the interceptor must close it. + */ + fun intercept( + image: ImageProxy, + isMirrored: Boolean, + ): ImageProxy? + + /** Called on the UI thread when a PreviewView is created. */ + fun onPreviewViewCreated(view: PreviewView) +} + +object FrameInterceptors { + private const val TAG = "FrameInterceptors" + + /** Whether this build of VisionCamera accepts an interceptor (decided at compile time). */ + val isEnabled: Boolean + get() = BuildConfig.FRAME_INTERCEPTORS + + @Volatile + private var registered: NativeFrameInterceptor? = null + + /** + * Registers [interceptor]. Only one interceptor can be registered, and only if [isEnabled]; + * otherwise the call is refused, logged and returns `false`. + */ + @Synchronized + fun register(interceptor: NativeFrameInterceptor): Boolean { + val name = interceptor.javaClass.name + if (!isEnabled) { + Log.e(TAG, "Frame interceptors are disabled in this build; ignoring $name.") + return false + } + registered?.let { + Log.e(TAG, "Frame interceptor ${it.javaClass.name} is already registered; ignoring $name.") + return false + } + registered = interceptor + Log.w(TAG, "Frame interceptor $name registered: camera frames and previews can be replaced in this build.") + return true + } + + internal fun intercept( + image: ImageProxy, + isMirrored: Boolean, + ): ImageProxy? { + val interceptor = registered ?: return image + return try { + interceptor.intercept(image, isMirrored) + } catch (e: Throwable) { + Log.e(TAG, "Frame interceptor failed; delivering the camera frame.", e) + image + } + } + + internal fun previewCreated(view: PreviewView) { + try { + registered?.onPreviewViewCreated(view) + } catch (e: Throwable) { + Log.e(TAG, "Frame interceptor failed to decorate a PreviewView.", e) + } + } +} diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/views/HybridPreviewView.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/views/HybridPreviewView.kt index dcbc15ded7..64bb174bf9 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/views/HybridPreviewView.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/views/HybridPreviewView.kt @@ -23,6 +23,7 @@ import com.margelo.nitro.camera.extensions.converters.toImplementationMode import com.margelo.nitro.camera.extensions.converters.toScaleType import com.margelo.nitro.camera.extensions.installHierarchyFitter import com.margelo.nitro.camera.hybrids.metering.HybridMeteringPoint +import com.margelo.nitro.camera.public.FrameInterceptors import com.margelo.nitro.camera.public.NativeGestureController import com.margelo.nitro.camera.public.NativePreviewOutput import com.margelo.nitro.core.Promise @@ -98,6 +99,7 @@ class HybridPreviewView( previewView.setOnTouchListener(this) meteringPointFactory = previewView.meteringPointFactory pixelRatio = context.resources.displayMetrics.density + FrameInterceptors.previewCreated(previewView) } override fun dispose() { diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift index 17c7a5f13c..872ba4ddb5 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift @@ -157,6 +157,7 @@ final class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOu at: timestamp, orientation: bufferOrientation, isMirrored: isBufferMirrored) + guard let sampleBuffer = FrameInterceptors.intercept(sampleBuffer, orientation: metadata.orientation, isMirrored: metadata.isMirrored) else { return } let frame = HybridFrame( buffer: sampleBuffer, metadata: metadata) diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Views/HybridPreviewView.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Views/HybridPreviewView.swift index 00d753251e..ef221b6b60 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Views/HybridPreviewView.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Views/HybridPreviewView.swift @@ -133,6 +133,7 @@ final class HybridPreviewView: HybridPreviewViewSpec { self.view.layer.addSublayer(previewLayer) } previewLayer.frame = self.view.bounds + FrameInterceptors.previewAttached(self.view) // Update resizeMode let resizeMode = self.resizeMode ?? .cover diff --git a/packages/react-native-vision-camera/ios/Public/FrameInterceptors.swift b/packages/react-native-vision-camera/ios/Public/FrameInterceptors.swift new file mode 100644 index 0000000000..419e9e39fd --- /dev/null +++ b/packages/react-native-vision-camera/ios/Public/FrameInterceptors.swift @@ -0,0 +1,75 @@ +/// +/// FrameInterceptors.swift +/// VisionCamera +/// +/// Optional extension point that decorates FrameOutput frame delivery and PreviewView attachment +/// (e.g. to inject test images). Interceptors are only accepted in builds compiled with +/// VISION_CAMERA_FRAME_INTERCEPTORS, set for the configurations listed in the app's package.json +/// under "visionCamera.frameInterceptors.ios" (default: Debug). While nothing is registered, +/// FrameOutput and PreviewView behave as usual. +/// + +import AVFoundation +import Foundation +import UIKit + +public protocol NativeFrameInterceptor: AnyObject { + /// Called on a FrameOutput's queue for every camera frame. Returns the frame to deliver: the + /// camera's `sampleBuffer`, a replacement, or nil to drop it. + func intercept( + _ sampleBuffer: CMSampleBuffer, orientation: CameraOrientation, isMirrored: Bool + ) -> CMSampleBuffer? + /// Called on the main thread after a PreviewView attached its preview layer to `view`. + func previewViewDidAttach(_ view: UIView) +} + +public enum FrameInterceptors { + /// Whether this build of VisionCamera accepts an interceptor (decided at compile time). + public static var isEnabled: Bool { + #if VISION_CAMERA_FRAME_INTERCEPTORS + return true + #else + return false + #endif + } + + private static let lock = NSLock() + private static var registered: (any NativeFrameInterceptor)? + + /// Registers `interceptor`. Only one interceptor can be registered, and only if `isEnabled`; + /// otherwise the call is refused, logged and returns `false`. + @discardableResult + public static func register(_ interceptor: any NativeFrameInterceptor) -> Bool { + let name = String(describing: type(of: interceptor)) + guard isEnabled else { + logger.error("Frame interceptors are disabled in this build; ignoring \(name).") + return false + } + lock.lock() + defer { lock.unlock() } + if let current = registered { + logger.error("Frame interceptor \(String(describing: type(of: current))) is already registered; ignoring \(name).") + return false + } + registered = interceptor + logger.warning("Frame interceptor \(name) registered: camera frames and previews can be replaced in this build.") + return true + } + + private static var current: (any NativeFrameInterceptor)? { + lock.lock() + defer { lock.unlock() } + return registered + } + + static func intercept( + _ sampleBuffer: CMSampleBuffer, orientation: CameraOrientation, isMirrored: Bool + ) -> CMSampleBuffer? { + guard let interceptor = current else { return sampleBuffer } + return interceptor.intercept(sampleBuffer, orientation: orientation, isMirrored: isMirrored) + } + + static func previewAttached(_ view: UIView) { + current?.previewViewDidAttach(view) + } +}