Multi-Camera Capture on iOS: The Architecture No One Tells You About

let backSession = AVCaptureSession()
let frontSession = AVCaptureSession()
// Configure separately, timestamp independently

Two AVCaptureSession instances
    ↓
Each timestamps frames independently
    ↓
Software attempts to align timestamps
    ↓
Drift compounds over time
    ↓
Resource contention (two IOSurface pools, fragmented memory)
    ↓
Result: 18-42ms drift, thermal runaway, memory pressure

Single AVCaptureMultiCamSession
    ↓
Multiple AVCaptureDeviceInput instances
    ↓
Hardware-synchronized AVCaptureConnection
    ↓
ISP-level timestamp coherence
    ↓
Shared CVMetalTextureCache (unified memory management)
    ↓
Single Metal command queue (GPUs aren't parallel across cameras)
    ↓
Coordinated thermal + lifecycle management
    ↓
Result: <5ms drift, sustainable thermal profile, predictable memory

guard AVCaptureMultiCamSession.isMultiCamSupported else {
    logger.warning("Multi-cam not supported on this device")
    return setupSingleCameraFallback()
}

func validateDeviceCombination(
    _ devices: [AVCaptureDevice],
    session: AVCaptureMultiCamSession
) -> Bool {
    let inputs = devices.compactMap {
        try? AVCaptureDeviceInput(device: $0)
    }

    // Test each input individually
    for input in inputs {
        guard session.canAddInput(input) else {
            logger.error("Cannot add input: \(input.device.localizedName)")
            return false
        }
    }

    return true
}

final class MultiCameraManager: NSObject {

    // MARK: - Session and Devices

    private let captureSession = AVCaptureMultiCamSession()

    // CRITICAL: Separate queue per camera output
    // Shared queue = dropped frames at 60 callbacks/sec (30 FPS × 2 cameras)
    private let backCameraQueue = DispatchQueue(
        label: "com.yourapp.camera.back",
        qos: .userInitiated
    )

    private let frontCameraQueue = DispatchQueue(
        label: "com.yourapp.camera.front",
        qos: .userInitiated
    )

    // Shared texture cache (memory efficiency)
    // Same pattern from Edition #11, now serving BOTH cameras
    private var textureCache: CVMetalTextureCache!
    private let metalDevice = MTLCreateSystemDefaultDevice()!

    private var backCameraOutput: AVCaptureVideoDataOutput?
    private var frontCameraOutput: AVCaptureVideoDataOutput?

    // MARK: - Configuration

    func configure() throws {
        // Create shared Metal texture cache
        // (Edition #11 pattern, now critical for dual cameras)
        var cache: CVMetalTextureCache?
        let result = CVMetalTextureCacheCreate(
            kCFAllocatorDefault,
            nil,
            metalDevice,
            nil,
            &cache
        )

        guard result == kCVReturnSuccess, let cache else {
            throw CameraError.metalSetupFailed
        }
        self.textureCache = cache

        captureSession.beginConfiguration()
        defer { captureSession.commitConfiguration() }

        // Configure back camera
        guard let backCamera = AVCaptureDevice.default(
            .builtInWideAngleCamera,
            for: .video,
            position: .back
        ) else {
            throw CameraError.deviceNotFound
        }

        try addCamera(
            backCamera,
            to: captureSession,
            output: &backCameraOutput,
            queue: backCameraQueue
        )

        // Configure front camera
        guard let frontCamera = AVCaptureDevice.default(
            .builtInWideAngleCamera,
            for: .video,
            position: .front
        ) else {
            throw CameraError.deviceNotFound
        }

        try addCamera(
            frontCamera,
            to: captureSession,
            output: &frontCameraOutput,
            queue: frontCameraQueue
        )

        logger.info("Multi-camera session configured successfully")
    }

    private func addCamera(
        _ device: AVCaptureDevice,
        to session: AVCaptureMultiCamSession,
        output: inout AVCaptureVideoDataOutput?,
        queue: DispatchQueue
    ) throws {
        let input = try AVCaptureDeviceInput(device: device)

        guard session.canAddInput(input) else {
            throw CameraError.cannotAddInput
        }
        session.addInput(input)

        let videoOutput = AVCaptureVideoDataOutput()

        // CRITICAL: Use YUV 420f (native camera format)
        // BGRA conversion costs ~2-3ms per frame + thermal overhead
        // This is the same format from Edition #11
        videoOutput.videoSettings = [
            kCVPixelBufferPixelFormatTypeKey as String:
                kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
        ]

        // Real-time processing: drop late frames to prevent latency buildup
        // Recording: set false and handle buffering in delegate
        videoOutput.alwaysDiscardsLateVideoFrames = true

        guard session.canAddOutput(videoOutput) else {
            throw CameraError.cannotAddOutput
        }
        session.addOutput(videoOutput)

        videoOutput.setSampleBufferDelegate(self, queue: queue)

        // Configure connection for hardware sync
        if let connection = videoOutput.connection(with: .video) {
            // Disable automatic mirroring (we control this)
            if connection.isVideoMirroringSupported {
                connection.automaticallyAdjustsVideoMirroring = false
            }

            // Set orientation
            if connection.isVideoOrientationSupported {
                connection.videoOrientation = .portrait
            }
        }

        output = videoOutput
    }
}

// MARK: - Frame Wrapper (Swift 6 Concurrency-Safe)

/// Wraps CVPixelBuffer for safe transfer across concurrency domains
/// @unchecked Sendable because CVPixelBuffer predates Sendable protocol
/// but is thread-safe via ARC (toll-free bridged to Swift)
struct CameraFrame: @unchecked Sendable {
    let pixelBuffer: CVPixelBuffer
    let timestamp: CMTime
    let position: AVCaptureDevice.Position

    // Note: CVPixelBuffer is ARC-managed in Swift
    // No manual CVBufferRetain/Release needed
}

// MARK: - Delegate Implementation

extension MultiCameraManager: AVCaptureVideoDataOutputSampleBufferDelegate {

    func captureOutput(
        _ output: AVCaptureOutput,
        didOutput sampleBuffer: CMSampleBuffer,
        from connection: AVCaptureConnection
    ) {
        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
            return
        }

        let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)

        // Determine camera position from connection
        let position: AVCaptureDevice.Position
        if let port = connection.inputPorts.first {
            position = port.sourceDevicePosition
        } else {
            logger.error("Cannot determine camera position")
            return
        }

        let frame = CameraFrame(
            pixelBuffer: pixelBuffer,
            timestamp: timestamp,
            position: position
        )

        // Track frame drift (this is how we measured the TestFlight data)
        trackFrameDrift(frame)

        // Process frame (Metal rendering, recording, etc.)
        processFrame(frame)
    }

    // MARK: - Frame Drift Tracking

    private var lastBackTimestamp: CMTime?
    private var lastFrontTimestamp: CMTime?

    private func trackFrameDrift(_ frame: CameraFrame) {
        switch frame.position {
        case .back:
            lastBackTimestamp = frame.timestamp
        case .front:
            lastFrontTimestamp = frame.timestamp
        default:
            break
        }

        // Calculate drift when we have both timestamps
        if let backTime = lastBackTimestamp,
           let frontTime = lastFrontTimestamp {
            let drift = abs(backTime.seconds - frontTime.seconds)

            if drift > 0.005 { // 5ms threshold
                logger.warning("Frame drift: \(drift * 1000)ms")
                metrics.maxFrameDrift = max(metrics.maxFrameDrift, drift)
            }
        }
    }

    // MARK: - Frame Processing

    private func processFrame(_ frame: CameraFrame) {
        // Convert YUV to Metal textures (zero-copy, same as Edition #11)
        guard let textures = createMetalTextures(from: frame.pixelBuffer) else {
            logger.error("Failed to create Metal textures")
            return
        }

        // Process with Metal shader (YUV → RGB conversion if needed)
        // Record to file, display to screen, etc.

        // CRITICAL: Flush texture cache periodically
        // (More critical with 2 cameras than single camera)
        frameCount += 1
        if frameCount % 60 == 0 {
            CVMetalTextureCacheFlush(textureCache, 0)
        }
    }

    // MARK: - Metal Texture Creation (Edition #11 pattern)

    private func createMetalTextures(
        from pixelBuffer: CVPixelBuffer
    ) -> (y: MTLTexture, uv: MTLTexture)? {

        let width = CVPixelBufferGetWidth(pixelBuffer)
        let height = CVPixelBufferGetHeight(pixelBuffer)

        // Y plane (luminance)
        var yTexture: CVMetalTexture?
        var result = CVMetalTextureCacheCreateTextureFromImage(
            kCFAllocatorDefault,
            textureCache,
            pixelBuffer,
            nil,
            .r8Unorm,
            width,
            height,
            0, // plane index
            &yTexture
        )

        guard result == kCVReturnSuccess,
              let yTex = yTexture,
              let yMetal = CVMetalTextureGetTexture(yTex) else {
            return nil
        }

        // UV plane (chrominance)
        var uvTexture: CVMetalTexture?
        result = CVMetalTextureCacheCreateTextureFromImage(
            kCFAllocatorDefault,
            textureCache,
            pixelBuffer,
            nil,
            .rg8Unorm,
            width / 2,
            height / 2,
            1, // plane index
            &uvTexture
        )

        guard result == kCVReturnSuccess,
              let uvTex = uvTexture,
              let uvMetal = CVMetalTextureGetTexture(uvTex) else {
            return nil
        }

        return (yMetal, uvMetal)
    }
}

@MainActor
final class ThermalManager {

    enum CameraMode {
        case dual           // Both cameras, full quality
        case backOnly       // Front suspended
        case reducedQuality // 720p, both cameras
        case minimal        // 480p, back only
    }

    private(set) var currentMode = CameraMode.dual

    // Don't react to transient spikes
    private var thermalHistory: [ProcessInfo.ThermalState] = []
    private let historyWindow = 5

    func handleThermalStateChange(_ state: ProcessInfo.ThermalState) -> CameraMode {
        thermalHistory.append(state)
        if thermalHistory.count > historyWindow {
            thermalHistory.removeFirst()
        }

        // Count serious/critical states in recent history
        let recentHigh = thermalHistory.filter {
            $0 == .serious || $0 == .critical
        }.count

        let newMode: CameraMode

        switch state {
        case .nominal, .fair:
            // Safe to run full quality
            newMode = .dual

        case .serious:
            // Persistent serious state → more aggressive throttling
            if recentHigh >= 3 {
                newMode = .backOnly
            } else {
                newMode = .reducedQuality
            }

        case .critical:
            // Emergency fallback
            newMode = .minimal

        @unknown default:
            newMode = .reducedQuality
        }

        if newMode != currentMode {
            currentMode = newMode
            logger.info("Thermal \(state) → mode \(newMode)")
        }

        return newMode
    }
}

// MARK: - Integration with Session

extension MultiCameraManager {

    func setupThermalMonitoring() {
        NotificationCenter.default.addObserver(
            forName: ProcessInfo.thermalStateDidChangeNotification,
            object: nil,
            queue: .main
        ) { [weak self] _ in
            guard let self else { return }
            let newMode = self.thermalManager.handleThermalStateChange(
                ProcessInfo.processInfo.thermalState
            )
            self.adjustCameraMode(newMode)
        }
    }

    private func adjustCameraMode(_ mode: ThermalManager.CameraMode) {
        captureSession.beginConfiguration()
        defer { captureSession.commitConfiguration() }

        switch mode {
        case .dual:
            // Full quality, both cameras
            if let front = frontCameraOutput {
                front.setSampleBufferDelegate(self, queue: frontCameraQueue)
            }
            captureSession.sessionPreset = .hd1920x1080

        case .backOnly:
            // Suspend front camera
            frontCameraOutput?.setSampleBufferDelegate(nil, queue: nil)
            captureSession.sessionPreset = .hd1920x1080

        case .reducedQuality:
            // 720p, both cameras
            if let front = frontCameraOutput {
                front.setSampleBufferDelegate(self, queue: frontCameraQueue)
            }
            captureSession.sessionPreset = .hd1280x720

        case .minimal:
            // Emergency: 480p, back only
            frontCameraOutput?.setSampleBufferDelegate(nil, queue: nil)
            captureSession.sessionPreset = .vga640x480
        }
    }
}

extension MultiCameraManager {

    func setupLifecycleMonitoring() {
        // KVO is more reliable than notifications for session state
        captureSession.observe(\.isRunning, options: [.new]) { [weak self] session, change in
            if let isRunning = change.newValue {
                if isRunning {
                    self?.handleSessionResumed()
                } else {
                    self?.handleSessionStopped()
                }
            }
        }

        // Monitor interruptions
        NotificationCenter.default.addObserver(
            forName: .AVCaptureSessionWasInterrupted,
            object: captureSession,
            queue: .main
        ) { [weak self] notification in
            if let reason = notification.userInfo?[AVCaptureSessionInterruptionReasonKey]
                as? AVCaptureSession.InterruptionReason {
                self?.handleInterruption(reason)
            }
        }

        NotificationCenter.default.addObserver(
            forName: .AVCaptureSessionInterruptionEnded,
            object: captureSession,
            queue: .main
        ) { [weak self] _ in
            self?.verifyBothCamerasAfterResume()
        }
    }

    private func handleInterruption(_ reason: AVCaptureSession.InterruptionReason) {
        switch reason {
        case .videoDeviceNotAvailableInBackground:
            logger.info("App backgrounded")

        case .videoDeviceInUseByAnotherClient:
            logger.warning("Camera taken by another app")

        case .videoDeviceNotAvailableWithMultipleForegroundApps:
            logger.warning("Split-screen/PiP active")

        @unknown default:
            logger.warning("Unknown interruption")
        }
    }

    // CRITICAL: This is the pattern that fixed our 67% → 98% resume success rate
    private func verifyBothCamerasAfterResume() {
        // Check both cameras individually
        // One may resume while the other stays interrupted

        var backActive = false
        var frontActive = false

        if let backConn = backCameraOutput?.connection(with: .video),
           backConn.isActive {
            backActive = true
        }

        if let frontConn = frontCameraOutput?.connection(with: .video),
           frontConn.isActive {
            frontActive = true
        }

        if !backActive || !frontActive {
            logger.error("""
                Camera state mismatch after resume:
                Back: \(backActive), Front: \(frontActive)
                Attempting recovery...
                """)

            // Attempt recovery
            if !captureSession.isRunning {
                captureSession.startRunning()
            }

            // Verify after short delay
            DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { [weak self] in
                guard let self else { return }

                if !self.captureSession.isRunning {
                    logger.error("Session failed to resume — falling back to single camera")
                    self.fallbackToSingleCamera()
                }
            }
        } else {
            logger.info("Both cameras resumed successfully")
        }
    }

    private func fallbackToSingleCamera() {
        // Graceful degradation to single-camera mode
        // (Implementation depends on app requirements)
        captureSession.beginConfiguration()
        defer { captureSession.commitConfiguration() }

        frontCameraOutput?.setSampleBufferDelegate(nil, queue: nil)
        logger.info("Running in single-camera fallback mode")
    }
}

struct CameraMetrics {
    var sessionStartTime: Date
    var framesProcessed: Int = 0
    var framesDropped: Int = 0
    var thermalStateChanges: [(ProcessInfo.ThermalState, Date)] = []
    var memoryWarnings: Int = 0
    var interruptionCount: Int = 0
    var maxFrameDrift: TimeInterval = 0

    func report() -> [String: Any] {
        let duration = Date().timeIntervalSince(sessionStartTime)
        let dropRate = Double(framesDropped) / Double(max(framesProcessed, 1))

        return [
            "session_duration_seconds": duration,
            "frames_processed": framesProcessed,
            "drop_rate_percent": dropRate * 100,
            "thermal_changes": thermalStateChanges.count,
            "memory_warnings": memoryWarnings,
            "max_frame_drift_ms": maxFrameDrift * 1000,
            "interruptions": interruptionCount
        ]
    }
}

// ❌ WRONG — creates new buffers, fragments memory
let newBuffer = CVPixelBuffer.create(...)

// ✅ RIGHT — reuse camera's IOSurface-backed buffers
// Just process the buffer you receive in the delegate