| /* |
| * Copyright 2016 The WebRTC project authors. All Rights Reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #import "RTCAVFoundationVideoCapturerInternal.h" |
| |
| #import <Foundation/Foundation.h> |
| #if TARGET_OS_IPHONE |
| #import <UIKit/UIKit.h> |
| #import "WebRTC/UIDevice+RTCDevice.h" |
| #endif |
| |
| #import "AVCaptureSession+DevicePosition.h" |
| #import "RTCDispatcher+Private.h" |
| #import "WebRTC/RTCLogging.h" |
| |
| #include "avfoundationformatmapper.h" |
| |
| @implementation RTCAVFoundationVideoCapturerInternal { |
| // Keep pointers to inputs for convenience. |
| AVCaptureDeviceInput *_frontCameraInput; |
| AVCaptureDeviceInput *_backCameraInput; |
| AVCaptureVideoDataOutput *_videoDataOutput; |
| // The cricket::VideoCapturer that owns this class. Should never be NULL. |
| webrtc::AVFoundationVideoCapturer *_capturer; |
| BOOL _hasRetriedOnFatalError; |
| BOOL _isRunning; |
| BOOL _hasStarted; |
| rtc::CriticalSection _crit; |
| #if TARGET_OS_IPHONE |
| UIDeviceOrientation _orientation; |
| #endif |
| } |
| |
| @synthesize captureSession = _captureSession; |
| @synthesize frameQueue = _frameQueue; |
| @synthesize useBackCamera = _useBackCamera; |
| |
| @synthesize isRunning = _isRunning; |
| @synthesize hasStarted = _hasStarted; |
| |
| // This is called from the thread that creates the video source, which is likely |
| // the main thread. |
| - (instancetype)initWithCapturer:(webrtc::AVFoundationVideoCapturer *)capturer { |
| RTC_DCHECK(capturer); |
| if (self = [super init]) { |
| _capturer = capturer; |
| // Create the capture session and all relevant inputs and outputs. We need |
| // to do this in init because the application may want the capture session |
| // before we start the capturer for e.g. AVCapturePreviewLayer. All objects |
| // created here are retained until dealloc and never recreated. |
| if (![self setupCaptureSession]) { |
| return nil; |
| } |
| NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; |
| #if TARGET_OS_IPHONE |
| _orientation = UIDeviceOrientationPortrait; |
| [center addObserver:self |
| selector:@selector(deviceOrientationDidChange:) |
| name:UIDeviceOrientationDidChangeNotification |
| object:nil]; |
| [center addObserver:self |
| selector:@selector(handleCaptureSessionInterruption:) |
| name:AVCaptureSessionWasInterruptedNotification |
| object:_captureSession]; |
| [center addObserver:self |
| selector:@selector(handleCaptureSessionInterruptionEnded:) |
| name:AVCaptureSessionInterruptionEndedNotification |
| object:_captureSession]; |
| [center addObserver:self |
| selector:@selector(handleApplicationDidBecomeActive:) |
| name:UIApplicationDidBecomeActiveNotification |
| object:[UIApplication sharedApplication]]; |
| #endif |
| [center addObserver:self |
| selector:@selector(handleCaptureSessionRuntimeError:) |
| name:AVCaptureSessionRuntimeErrorNotification |
| object:_captureSession]; |
| [center addObserver:self |
| selector:@selector(handleCaptureSessionDidStartRunning:) |
| name:AVCaptureSessionDidStartRunningNotification |
| object:_captureSession]; |
| [center addObserver:self |
| selector:@selector(handleCaptureSessionDidStopRunning:) |
| name:AVCaptureSessionDidStopRunningNotification |
| object:_captureSession]; |
| } |
| return self; |
| } |
| |
| - (void)dealloc { |
| RTC_DCHECK(!self.hasStarted); |
| [[NSNotificationCenter defaultCenter] removeObserver:self]; |
| _capturer = nullptr; |
| } |
| |
| - (AVCaptureSession *)captureSession { |
| return _captureSession; |
| } |
| |
| - (AVCaptureDevice *)getActiveCaptureDevice { |
| return self.useBackCamera ? _backCameraInput.device : _frontCameraInput.device; |
| } |
| |
| - (nullable AVCaptureDevice *)frontCaptureDevice { |
| return _frontCameraInput.device; |
| } |
| |
| - (nullable AVCaptureDevice *)backCaptureDevice { |
| return _backCameraInput.device; |
| } |
| |
| - (dispatch_queue_t)frameQueue { |
| if (!_frameQueue) { |
| _frameQueue = |
| dispatch_queue_create("org.webrtc.avfoundationvideocapturer.video", DISPATCH_QUEUE_SERIAL); |
| dispatch_set_target_queue(_frameQueue, |
| dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0)); |
| } |
| return _frameQueue; |
| } |
| |
| // Called from any thread (likely main thread). |
| - (BOOL)canUseBackCamera { |
| return _backCameraInput != nil; |
| } |
| |
| // Called from any thread (likely main thread). |
| - (BOOL)useBackCamera { |
| @synchronized(self) { |
| return _useBackCamera; |
| } |
| } |
| |
| // Called from any thread (likely main thread). |
| - (void)setUseBackCamera:(BOOL)useBackCamera { |
| if (!self.canUseBackCamera) { |
| if (useBackCamera) { |
| RTCLogWarning(@"No rear-facing camera exists or it cannot be used;" |
| "not switching."); |
| } |
| return; |
| } |
| @synchronized(self) { |
| if (_useBackCamera == useBackCamera) { |
| return; |
| } |
| _useBackCamera = useBackCamera; |
| [self updateSessionInputForUseBackCamera:useBackCamera]; |
| } |
| } |
| |
| // Called from WebRTC thread. |
| - (void)start { |
| if (self.hasStarted) { |
| return; |
| } |
| self.hasStarted = YES; |
| [RTCDispatcher |
| dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| [self updateOrientation]; |
| #if TARGET_OS_IPHONE |
| [[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications]; |
| #endif |
| AVCaptureSession *captureSession = self.captureSession; |
| [captureSession startRunning]; |
| }]; |
| } |
| |
| // Called from same thread as start. |
| - (void)stop { |
| if (!self.hasStarted) { |
| return; |
| } |
| self.hasStarted = NO; |
| // Due to this async block, it's possible that the ObjC object outlives the |
| // C++ one. In order to not invoke functions on the C++ object, we set |
| // hasStarted immediately instead of dispatching it async. |
| [RTCDispatcher |
| dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| [_videoDataOutput setSampleBufferDelegate:nil queue:nullptr]; |
| [_captureSession stopRunning]; |
| #if TARGET_OS_IPHONE |
| [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications]; |
| #endif |
| }]; |
| } |
| |
| #pragma mark iOS notifications |
| |
| #if TARGET_OS_IPHONE |
| - (void)deviceOrientationDidChange:(NSNotification *)notification { |
| [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| [self updateOrientation]; |
| }]; |
| } |
| #endif |
| |
| #pragma mark AVCaptureVideoDataOutputSampleBufferDelegate |
| |
| - (void)captureOutput:(AVCaptureOutput *)captureOutput |
| didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer |
| fromConnection:(AVCaptureConnection *)connection { |
| NSParameterAssert(captureOutput == _videoDataOutput); |
| if (!self.hasStarted) { |
| return; |
| } |
| |
| #if TARGET_OS_IPHONE |
| // Default to portrait orientation on iPhone. |
| webrtc::VideoRotation rotation = webrtc::kVideoRotation_90; |
| BOOL usingFrontCamera = NO; |
| // Check the image's EXIF for the camera the image came from as the image could have been |
| // delayed as we set alwaysDiscardsLateVideoFrames to NO. |
| AVCaptureDevicePosition cameraPosition = |
| [AVCaptureSession devicePositionForSampleBuffer:sampleBuffer]; |
| if (cameraPosition != AVCaptureDevicePositionUnspecified) { |
| usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition; |
| } else { |
| AVCaptureDeviceInput *deviceInput = |
| (AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input; |
| usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position; |
| } |
| switch (_orientation) { |
| case UIDeviceOrientationPortrait: |
| rotation = webrtc::kVideoRotation_90; |
| break; |
| case UIDeviceOrientationPortraitUpsideDown: |
| rotation = webrtc::kVideoRotation_270; |
| break; |
| case UIDeviceOrientationLandscapeLeft: |
| rotation = usingFrontCamera ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0; |
| break; |
| case UIDeviceOrientationLandscapeRight: |
| rotation = usingFrontCamera ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180; |
| break; |
| case UIDeviceOrientationFaceUp: |
| case UIDeviceOrientationFaceDown: |
| case UIDeviceOrientationUnknown: |
| // Ignore. |
| break; |
| } |
| #else |
| // No rotation on Mac. |
| webrtc::VideoRotation rotation = webrtc::kVideoRotation_0; |
| #endif |
| |
| _capturer->CaptureSampleBuffer(sampleBuffer, rotation); |
| } |
| |
| - (void)captureOutput:(AVCaptureOutput *)captureOutput |
| didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer |
| fromConnection:(AVCaptureConnection *)connection { |
| RTCLogError(@"Dropped sample buffer."); |
| } |
| |
| #pragma mark - AVCaptureSession notifications |
| |
| - (void)handleCaptureSessionInterruption:(NSNotification *)notification { |
| NSString *reasonString = nil; |
| #if TARGET_OS_IPHONE |
| NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey]; |
| if (reason) { |
| switch (reason.intValue) { |
| case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground: |
| reasonString = @"VideoDeviceNotAvailableInBackground"; |
| break; |
| case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient: |
| reasonString = @"AudioDeviceInUseByAnotherClient"; |
| break; |
| case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient: |
| reasonString = @"VideoDeviceInUseByAnotherClient"; |
| break; |
| case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps: |
| reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps"; |
| break; |
| } |
| } |
| #endif |
| RTCLog(@"Capture session interrupted: %@", reasonString); |
| // TODO(tkchin): Handle this case. |
| } |
| |
| - (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification { |
| RTCLog(@"Capture session interruption ended."); |
| // TODO(tkchin): Handle this case. |
| } |
| |
| - (void)handleCaptureSessionRuntimeError:(NSNotification *)notification { |
| NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey]; |
| RTCLogError(@"Capture session runtime error: %@", error); |
| |
| [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| #if TARGET_OS_IPHONE |
| if (error.code == AVErrorMediaServicesWereReset) { |
| [self handleNonFatalError]; |
| } else { |
| [self handleFatalError]; |
| } |
| #else |
| [self handleFatalError]; |
| #endif |
| }]; |
| } |
| |
| - (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification { |
| RTCLog(@"Capture session started."); |
| |
| self.isRunning = YES; |
| [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| // If we successfully restarted after an unknown error, |
| // allow future retries on fatal errors. |
| _hasRetriedOnFatalError = NO; |
| }]; |
| } |
| |
| - (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification { |
| RTCLog(@"Capture session stopped."); |
| self.isRunning = NO; |
| } |
| |
| - (void)handleFatalError { |
| [RTCDispatcher |
| dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| if (!_hasRetriedOnFatalError) { |
| RTCLogWarning(@"Attempting to recover from fatal capture error."); |
| [self handleNonFatalError]; |
| _hasRetriedOnFatalError = YES; |
| } else { |
| RTCLogError(@"Previous fatal error recovery failed."); |
| } |
| }]; |
| } |
| |
| - (void)handleNonFatalError { |
| [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| if (self.hasStarted) { |
| RTCLog(@"Restarting capture session after error."); |
| [self.captureSession startRunning]; |
| } |
| }]; |
| } |
| |
| #if TARGET_OS_IPHONE |
| |
| #pragma mark - UIApplication notifications |
| |
| - (void)handleApplicationDidBecomeActive:(NSNotification *)notification { |
| [RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| if (self.hasStarted && !self.captureSession.isRunning) { |
| RTCLog(@"Restarting capture session on active."); |
| [self.captureSession startRunning]; |
| } |
| }]; |
| } |
| |
| #endif // TARGET_OS_IPHONE |
| |
| #pragma mark - Private |
| |
| - (BOOL)setupCaptureSession { |
| AVCaptureSession *captureSession = [[AVCaptureSession alloc] init]; |
| #if defined(WEBRTC_IOS) |
| captureSession.usesApplicationAudioSession = NO; |
| #endif |
| // Add the output. |
| AVCaptureVideoDataOutput *videoDataOutput = [self videoDataOutput]; |
| if (![captureSession canAddOutput:videoDataOutput]) { |
| RTCLogError(@"Video data output unsupported."); |
| return NO; |
| } |
| [captureSession addOutput:videoDataOutput]; |
| |
| // Get the front and back cameras. If there isn't a front camera |
| // give up. |
| AVCaptureDeviceInput *frontCameraInput = [self frontCameraInput]; |
| AVCaptureDeviceInput *backCameraInput = [self backCameraInput]; |
| if (!frontCameraInput) { |
| RTCLogError(@"No front camera for capture session."); |
| return NO; |
| } |
| |
| // Add the inputs. |
| if (![captureSession canAddInput:frontCameraInput] || |
| (backCameraInput && ![captureSession canAddInput:backCameraInput])) { |
| RTCLogError(@"Session does not support capture inputs."); |
| return NO; |
| } |
| AVCaptureDeviceInput *input = self.useBackCamera ? backCameraInput : frontCameraInput; |
| [captureSession addInput:input]; |
| |
| _captureSession = captureSession; |
| return YES; |
| } |
| |
| - (AVCaptureVideoDataOutput *)videoDataOutput { |
| if (!_videoDataOutput) { |
| // Make the capturer output NV12. Ideally we want I420 but that's not |
| // currently supported on iPhone / iPad. |
| AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; |
| videoDataOutput.videoSettings = @{ |
| (NSString *) |
| // TODO(denicija): Remove this color conversion and use the original capture format directly. |
| kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) |
| }; |
| videoDataOutput.alwaysDiscardsLateVideoFrames = NO; |
| [videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue]; |
| _videoDataOutput = videoDataOutput; |
| } |
| return _videoDataOutput; |
| } |
| |
| - (AVCaptureDevice *)videoCaptureDeviceForPosition:(AVCaptureDevicePosition)position { |
| for (AVCaptureDevice *captureDevice in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) { |
| if (captureDevice.position == position) { |
| return captureDevice; |
| } |
| } |
| return nil; |
| } |
| |
| - (AVCaptureDeviceInput *)frontCameraInput { |
| if (!_frontCameraInput) { |
| #if TARGET_OS_IPHONE |
| AVCaptureDevice *frontCameraDevice = |
| [self videoCaptureDeviceForPosition:AVCaptureDevicePositionFront]; |
| #else |
| AVCaptureDevice *frontCameraDevice = |
| [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; |
| #endif |
| if (!frontCameraDevice) { |
| RTCLogWarning(@"Failed to find front capture device."); |
| return nil; |
| } |
| NSError *error = nil; |
| AVCaptureDeviceInput *frontCameraInput = |
| [AVCaptureDeviceInput deviceInputWithDevice:frontCameraDevice error:&error]; |
| if (!frontCameraInput) { |
| RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription); |
| return nil; |
| } |
| _frontCameraInput = frontCameraInput; |
| } |
| return _frontCameraInput; |
| } |
| |
| - (AVCaptureDeviceInput *)backCameraInput { |
| if (!_backCameraInput) { |
| AVCaptureDevice *backCameraDevice = |
| [self videoCaptureDeviceForPosition:AVCaptureDevicePositionBack]; |
| if (!backCameraDevice) { |
| RTCLogWarning(@"Failed to find front capture device."); |
| return nil; |
| } |
| NSError *error = nil; |
| AVCaptureDeviceInput *backCameraInput = |
| [AVCaptureDeviceInput deviceInputWithDevice:backCameraDevice error:&error]; |
| if (!backCameraInput) { |
| RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription); |
| return nil; |
| } |
| _backCameraInput = backCameraInput; |
| } |
| return _backCameraInput; |
| } |
| |
| // Called from capture session queue. |
| - (void)updateOrientation { |
| #if TARGET_OS_IPHONE |
| _orientation = [UIDevice currentDevice].orientation; |
| #endif |
| } |
| |
| // Update the current session input to match what's stored in _useBackCamera. |
| - (void)updateSessionInputForUseBackCamera:(BOOL)useBackCamera { |
| [RTCDispatcher |
| dispatchAsyncOnType:RTCDispatcherTypeCaptureSession |
| block:^{ |
| [_captureSession beginConfiguration]; |
| AVCaptureDeviceInput *oldInput = _backCameraInput; |
| AVCaptureDeviceInput *newInput = _frontCameraInput; |
| if (useBackCamera) { |
| oldInput = _frontCameraInput; |
| newInput = _backCameraInput; |
| } |
| if (oldInput) { |
| // Ok to remove this even if it's not attached. Will be no-op. |
| [_captureSession removeInput:oldInput]; |
| } |
| if (newInput) { |
| [_captureSession addInput:newInput]; |
| } |
| [self updateOrientation]; |
| AVCaptureDevice *newDevice = newInput.device; |
| const cricket::VideoFormat *format = _capturer->GetCaptureFormat(); |
| webrtc::SetFormatForCaptureDevice(newDevice, _captureSession, *format); |
| [_captureSession commitConfiguration]; |
| }]; |
| } |
| |
| @end |
