feat: Add depth to Frame

example/ios/Podfile.lock

@@ -1768,16 +1768,16 @@ PODS:
     - ReactCommon/turbomodule/core
     - Yoga
   - SocketRocket (0.7.0)
-  - VisionCamera (4.6.1):
-    - VisionCamera/Core (= 4.6.1)
-    - VisionCamera/FrameProcessors (= 4.6.1)
-    - VisionCamera/React (= 4.6.1)
-  - VisionCamera/Core (4.6.1)
-  - VisionCamera/FrameProcessors (4.6.1):
+  - VisionCamera (4.6.3):
+    - VisionCamera/Core (= 4.6.3)
+    - VisionCamera/FrameProcessors (= 4.6.3)
+    - VisionCamera/React (= 4.6.3)
+  - VisionCamera/Core (4.6.3)
+  - VisionCamera/FrameProcessors (4.6.3):
     - React
     - React-callinvoker
     - react-native-worklets-core
-  - VisionCamera/React (4.6.1):
+  - VisionCamera/React (4.6.3):
     - React-Core
     - VisionCamera/FrameProcessors
   - Yoga (0.0.0)
@@ -2097,9 +2097,9 @@ SPEC CHECKSUMS:
   RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
   RNVectorIcons: 6382277afab3c54658e9d555ee0faa7a37827136
   SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d
-  VisionCamera: ec141897a88c2e95e8b83cf97b8e4db801e02fd6
-  Yoga: 055f92ad73f8c8600a93f0e25ac0b2344c3b07e6
+  VisionCamera: 88df4dae7196c93ecd331f105f0e5d7d95702cb3
+  Yoga: aa3df615739504eebb91925fc9c58b4922ea9a08
 
 PODFILE CHECKSUM: 2ad84241179871ca890f7c65c855d117862f1a68
 
-COCOAPODS: 1.16.2
+COCOAPODS: 1.15.2

package/ios/Core/CameraConfiguration.swift

@@ -182,6 +182,7 @@ final class CameraConfiguration {
     var enableBufferCompression = false
     var enableHdr = false
     var enableFrameProcessor = false
+    var enableDepth = false
   }
 
   /**

package/ios/Core/CameraSession+Configuration.swift

@@ -54,7 +54,7 @@ extension CameraSession {
   // pragma MARK: Outputs
 
   /**
-   Configures all outputs (`photo` + `video` + `codeScanner`)
+   Configures all outputs (`photo` + `video` + `depth` + `codeScanner`)
    */
   func configureOutputs(configuration: CameraConfiguration) throws {
     VisionLogger.log(level: .info, message: "Configuring Outputs...")
@@ -65,7 +65,9 @@ extension CameraSession {
     }
     photoOutput = nil
     videoOutput = nil
+    depthOutput = nil
     codeScannerOutput = nil
+    outputSynchronizer = nil
 
     // Photo Output
     if case let .enabled(photo) = configuration.photo {
@@ -97,7 +99,7 @@ extension CameraSession {
     }
 
     // Video Output + Frame Processor
-    if case .enabled = configuration.video {
+    if case let .enabled(video) = configuration.video {
       VisionLogger.log(level: .info, message: "Adding Video Data output...")
 
       // 1. Add
@@ -107,8 +109,7 @@ extension CameraSession {
       }
       captureSession.addOutput(videoOutput)
 
-      // 2. Configure
-      videoOutput.setSampleBufferDelegate(self, queue: CameraQueues.videoQueue)
+      // 2. Configure Video
       videoOutput.alwaysDiscardsLateVideoFrames = true
       if configuration.isMirrored {
         // 2.1. If mirroring is enabled, mirror all connections along the vertical axis
@@ -120,6 +121,24 @@ extension CameraSession {
         }
       }
 
+      // 3. Configure Depth
+      if video.enableDepth {
+        // Video is synchronized with depth data - use a joined delegate!
+        // 3.1. Create depth output
+        let depthOutput = AVCaptureDepthDataOutput()
+        depthOutput.alwaysDiscardsLateDepthData = true
+        depthOutput.isFilteringEnabled = false
+        // 3.2. Add depth output to session
+        captureSession.addOutput(depthOutput)
+        // 3.3. Set up a synchronizer between video and depth data
+        outputSynchronizer = AVCaptureDataOutputSynchronizer(dataOutputs: [depthOutput, videoOutput])
+        outputSynchronizer!.setDelegate(self, queue: CameraQueues.videoQueue)
+        self.depthOutput = depthOutput
+      } else {
+        // Video is the only output - use it's own delegate
+        videoOutput.setSampleBufferDelegate(self, queue: CameraQueues.videoQueue)
+      }
+
       self.videoOutput = videoOutput
     }
 

package/ios/Core/CameraSession.swift

@@ -13,7 +13,11 @@ import Foundation
  A fully-featured Camera Session supporting preview, video, photo, frame processing, and code scanning outputs.
  All changes to the session have to be controlled via the `configure` function.
  */
-final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
+final class CameraSession:
+  NSObject,
+  AVCaptureVideoDataOutputSampleBufferDelegate,
+  AVCaptureAudioDataOutputSampleBufferDelegate,
+  AVCaptureDataOutputSynchronizerDelegate {
   // Configuration
   private var isInitialized = false
   var configuration: CameraConfiguration?
@@ -27,7 +31,9 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
   var photoOutput: AVCapturePhotoOutput?
   var videoOutput: AVCaptureVideoDataOutput?
   var audioOutput: AVCaptureAudioDataOutput?
+  var depthOutput: AVCaptureDepthDataOutput?
   var codeScannerOutput: AVCaptureMetadataOutput?
+  var outputSynchronizer: AVCaptureDataOutputSynchronizer?
   // State
   var metadataProvider = MetadataProvider()
   var recordingSession: RecordingSession?
@@ -276,7 +282,24 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
     }
   }
 
-  private final func onVideoFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool) {
+  func dataOutputSynchronizer(_: AVCaptureDataOutputSynchronizer, didOutput synchronizedDataCollection: AVCaptureSynchronizedDataCollection) {
+    guard let videoOutput else { return }
+    guard let videoData = synchronizedDataCollection.synchronizedData(for: videoOutput) as? AVCaptureSynchronizedSampleBufferData else { return }
+
+    if let depthOutput {
+      // We have depth data as well
+      guard let depthData = synchronizedDataCollection.synchronizedData(for: videoOutput) as? AVCaptureSynchronizedSampleBufferData else { return }
+      onVideoFrame(sampleBuffer: videoData.sampleBuffer,
+                   orientation: videoOutput.orientation,
+                   isMirrored: videoOutput.isMirrored,
+                   depthData: depthData.sampleBuffer)
+    } else {
+      // We only have video data
+      onVideoFrame(sampleBuffer: videoData.sampleBuffer, orientation: videoOutput.orientation, isMirrored: videoOutput.isMirrored)
+    }
+  }
+
+  private final func onVideoFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthData: CMSampleBuffer? = nil) {
     if let recordingSession {
       do {
         // Write the Video Buffer to the .mov/.mp4 file
@@ -290,7 +313,10 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
 
     if let delegate {
       // Call Frame Processor (delegate) for every Video Frame
-      delegate.onFrame(sampleBuffer: sampleBuffer, orientation: orientation, isMirrored: isMirrored)
+      delegate.onFrame(sampleBuffer: sampleBuffer,
+                       orientation: orientation,
+                       isMirrored: isMirrored,
+                       depthBuffer: depthData)
     }
   }
 

package/ios/Core/CameraSessionDelegate.swift

@@ -44,7 +44,7 @@ protocol CameraSessionDelegate: AnyObject {
   /**
    Called for every frame (if video or frameProcessor is enabled)
    */
-  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool)
+  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthBuffer: CMSampleBuffer?)
   /**
    Called whenever a QR/Barcode has been scanned. Only if the CodeScanner Output is enabled
    */

package/ios/FrameProcessors/Frame.h

@@ -16,13 +16,17 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface Frame : NSObject
 
-- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored;
+- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer
+                   orientation:(UIImageOrientation)orientation
+                    isMirrored:(BOOL)isMirrored
+                     depthData:(nullable CMSampleBufferRef)depth;
 - (instancetype)init NS_UNAVAILABLE;
 
 - (void)incrementRefCount;
 - (void)decrementRefCount;
 
 @property(nonatomic, readonly) CMSampleBufferRef buffer;
+@property(nonatomic, readonly, nullable) CMSampleBufferRef depth;
 @property(nonatomic, readonly) UIImageOrientation orientation;
 
 @property(nonatomic, readonly) NSString* pixelFormat;

package/ios/FrameProcessors/Frame.m

@@ -14,14 +14,19 @@ @implementation Frame {
   CMSampleBufferRef _Nonnull _buffer;
   UIImageOrientation _orientation;
   BOOL _isMirrored;
+  CMSampleBufferRef _Nullable _depth;
 }
 
-- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation isMirrored:(BOOL)isMirrored {
+- (instancetype)initWithBuffer:(CMSampleBufferRef)buffer
+                   orientation:(UIImageOrientation)orientation
+                    isMirrored:(BOOL)isMirrored
+                     depthData:(nullable CMSampleBufferRef)depth {
   self = [super init];
   if (self) {
     _buffer = buffer;
     _orientation = orientation;
     _isMirrored = isMirrored;
+    _depth = depth;
   }
   return self;
 }
@@ -47,6 +52,10 @@ - (CMSampleBufferRef)buffer {
   return _buffer;
 }
 
+- (nullable CMSampleBufferRef)depth {
+  return _depth;
+}
+
 - (BOOL)isValid {
   return _buffer != nil && CFGetRetainCount(_buffer) > 0 && CMSampleBufferIsValid(_buffer);
 }

package/ios/FrameProcessors/FrameHostObject.mm

@@ -74,6 +74,16 @@
   if (name == "planesCount") {
     return jsi::Value((double)_frame.planesCount);
   }
+  if (name == "depth") {
+    if (_frame.depth == nil) {
+      return jsi::Value::undefined();
+    }
+    jsi::Object object(runtime);
+    CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_frame.depth);
+    object.setProperty(runtime, "width", jsi::Value(static_cast<double>(CVPixelBufferGetWidth(imageBuffer))));
+    object.setProperty(runtime, "height", jsi::Value(static_cast<double>(CVPixelBufferGetHeight(imageBuffer))));
+    return object;
+  }
 
   // Internal methods
   if (name == "incrementRefCount") {

package/ios/React/CameraView.swift

@@ -211,7 +211,8 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat
         config.video = .enabled(config: CameraConfiguration.Video(pixelFormat: getPixelFormat(),
                                                                   enableBufferCompression: enableBufferCompression,
                                                                   enableHdr: videoHdr,
-                                                                  enableFrameProcessor: enableFrameProcessor))
+                                                                  enableFrameProcessor: enableFrameProcessor,
+                                                                  enableDepth: enableDepthData))
       } else {
         config.video = .disabled
       }
@@ -362,7 +363,7 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat
     ])
   }
 
-  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool) {
+  func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation, isMirrored: Bool, depthBuffer: CMSampleBuffer?) {
     // Update latest frame that can be used for snapshot capture
     latestVideoFrame = Snapshot(imageBuffer: sampleBuffer, orientation: orientation)
 
@@ -374,7 +375,8 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat
         // Call Frame Processor
         let frame = Frame(buffer: sampleBuffer,
                           orientation: orientation.imageOrientation,
-                          isMirrored: isMirrored)
+                          isMirrored: isMirrored,
+                          depthData: depthBuffer)
         frameProcessor.call(frame)
       }
     #endif

package/src/devices/getCameraFormat.ts

@@ -74,6 +74,10 @@ export interface FormatFilter {
    * you might want to choose a different auto-focus system.
    */
   autoFocusSystem?: AutoFocusSystem
+  /**
+   * Specifies whether to prefer formats that support depth data capture.
+   */
+  depth?: boolean
 }
 
 type FilterWithPriority<T> = {
@@ -236,6 +240,12 @@ export function getCameraFormat(device: CameraDevice, filters: FormatFilter[]):
       if (format.autoFocusSystem === filter.autoFocusSystem.target) rightPoints += filter.autoFocusSystem.priority
     }
 
+    // Find depth data
+    if (filter.depth != null) {
+      if (bestFormat.supportsDepthCapture) leftPoints += filter.depth.priority
+      if (format.supportsDepthCapture) rightPoints += filter.depth.priority
+    }
+
     if (rightPoints > leftPoints) bestFormat = format
   }
 

package/src/types/Frame.ts

@@ -64,6 +64,14 @@ export interface Frame {
    */
   readonly pixelFormat: PixelFormat
 
+  /**
+   * Represents the depth data of this Frame, if the Camera is configured to stream depth data.
+   */
+  readonly depth?: {
+    readonly width: number
+    readonly height: number
+  }
+
   /**
    * Get the underlying data of the Frame as a uint8 array buffer.
    *