blob: b3722b05510b24db0cabcb0a181a9fbb87c35a35 [file] [log] [blame]
/*
* Copyright 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "RTCAVFoundationVideoCapturerInternal.h"
#import <Foundation/Foundation.h>
#if TARGET_OS_IPHONE
#import <UIKit/UIKit.h>
#import "WebRTC/UIDevice+RTCDevice.h"
#endif
#import "AVCaptureSession+DevicePosition.h"
#import "RTCDispatcher+Private.h"
#import "WebRTC/RTCLogging.h"
#include "avfoundationformatmapper.h"
@implementation RTCAVFoundationVideoCapturerInternal {
// Keep pointers to inputs for convenience.
AVCaptureDeviceInput *_frontCameraInput;
AVCaptureDeviceInput *_backCameraInput;
AVCaptureVideoDataOutput *_videoDataOutput;
// The cricket::VideoCapturer that owns this class. Should never be NULL.
webrtc::AVFoundationVideoCapturer *_capturer;
BOOL _hasRetriedOnFatalError;
BOOL _isRunning;
BOOL _hasStarted;
rtc::CriticalSection _crit;
#if TARGET_OS_IPHONE
UIDeviceOrientation _orientation;
#endif
}
@synthesize captureSession = _captureSession;
@synthesize frameQueue = _frameQueue;
@synthesize useBackCamera = _useBackCamera;
@synthesize isRunning = _isRunning;
@synthesize hasStarted = _hasStarted;
// This is called from the thread that creates the video source, which is likely
// the main thread.
- (instancetype)initWithCapturer:(webrtc::AVFoundationVideoCapturer *)capturer {
RTC_DCHECK(capturer);
if (self = [super init]) {
_capturer = capturer;
// Create the capture session and all relevant inputs and outputs. We need
// to do this in init because the application may want the capture session
// before we start the capturer for e.g. AVCapturePreviewLayer. All objects
// created here are retained until dealloc and never recreated.
if (![self setupCaptureSession]) {
return nil;
}
NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
#if TARGET_OS_IPHONE
_orientation = UIDeviceOrientationPortrait;
[center addObserver:self
selector:@selector(deviceOrientationDidChange:)
name:UIDeviceOrientationDidChangeNotification
object:nil];
[center addObserver:self
selector:@selector(handleCaptureSessionInterruption:)
name:AVCaptureSessionWasInterruptedNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleCaptureSessionInterruptionEnded:)
name:AVCaptureSessionInterruptionEndedNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleApplicationDidBecomeActive:)
name:UIApplicationDidBecomeActiveNotification
object:[UIApplication sharedApplication]];
#endif
[center addObserver:self
selector:@selector(handleCaptureSessionRuntimeError:)
name:AVCaptureSessionRuntimeErrorNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleCaptureSessionDidStartRunning:)
name:AVCaptureSessionDidStartRunningNotification
object:_captureSession];
[center addObserver:self
selector:@selector(handleCaptureSessionDidStopRunning:)
name:AVCaptureSessionDidStopRunningNotification
object:_captureSession];
}
return self;
}
- (void)dealloc {
RTC_DCHECK(!self.hasStarted);
[[NSNotificationCenter defaultCenter] removeObserver:self];
_capturer = nullptr;
}
- (AVCaptureSession *)captureSession {
return _captureSession;
}
- (AVCaptureDevice *)getActiveCaptureDevice {
return self.useBackCamera ? _backCameraInput.device : _frontCameraInput.device;
}
- (nullable AVCaptureDevice *)frontCaptureDevice {
return _frontCameraInput.device;
}
- (nullable AVCaptureDevice *)backCaptureDevice {
return _backCameraInput.device;
}
- (dispatch_queue_t)frameQueue {
if (!_frameQueue) {
_frameQueue =
dispatch_queue_create("org.webrtc.avfoundationvideocapturer.video", DISPATCH_QUEUE_SERIAL);
dispatch_set_target_queue(_frameQueue,
dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0));
}
return _frameQueue;
}
// Called from any thread (likely main thread).
- (BOOL)canUseBackCamera {
return _backCameraInput != nil;
}
// Called from any thread (likely main thread).
- (BOOL)useBackCamera {
@synchronized(self) {
return _useBackCamera;
}
}
// Called from any thread (likely main thread).
- (void)setUseBackCamera:(BOOL)useBackCamera {
if (!self.canUseBackCamera) {
if (useBackCamera) {
RTCLogWarning(@"No rear-facing camera exists or it cannot be used;"
"not switching.");
}
return;
}
@synchronized(self) {
if (_useBackCamera == useBackCamera) {
return;
}
_useBackCamera = useBackCamera;
[self updateSessionInputForUseBackCamera:useBackCamera];
}
}
// Called from WebRTC thread.
- (void)start {
if (self.hasStarted) {
return;
}
self.hasStarted = YES;
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
[self updateOrientation];
#if TARGET_OS_IPHONE
[[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
#endif
AVCaptureSession *captureSession = self.captureSession;
[captureSession startRunning];
}];
}
// Called from same thread as start.
- (void)stop {
if (!self.hasStarted) {
return;
}
self.hasStarted = NO;
// Due to this async block, it's possible that the ObjC object outlives the
// C++ one. In order to not invoke functions on the C++ object, we set
// hasStarted immediately instead of dispatching it async.
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
[_videoDataOutput setSampleBufferDelegate:nil queue:nullptr];
[_captureSession stopRunning];
#if TARGET_OS_IPHONE
[[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
#endif
}];
}
#pragma mark iOS notifications
#if TARGET_OS_IPHONE
- (void)deviceOrientationDidChange:(NSNotification *)notification {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
[self updateOrientation];
}];
}
#endif
#pragma mark AVCaptureVideoDataOutputSampleBufferDelegate
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
NSParameterAssert(captureOutput == _videoDataOutput);
if (!self.hasStarted) {
return;
}
#if TARGET_OS_IPHONE
// Default to portrait orientation on iPhone.
webrtc::VideoRotation rotation = webrtc::kVideoRotation_90;
BOOL usingFrontCamera = NO;
// Check the image's EXIF for the camera the image came from as the image could have been
// delayed as we set alwaysDiscardsLateVideoFrames to NO.
AVCaptureDevicePosition cameraPosition =
[AVCaptureSession devicePositionForSampleBuffer:sampleBuffer];
if (cameraPosition != AVCaptureDevicePositionUnspecified) {
usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition;
} else {
AVCaptureDeviceInput *deviceInput =
(AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position;
}
switch (_orientation) {
case UIDeviceOrientationPortrait:
rotation = webrtc::kVideoRotation_90;
break;
case UIDeviceOrientationPortraitUpsideDown:
rotation = webrtc::kVideoRotation_270;
break;
case UIDeviceOrientationLandscapeLeft:
rotation = usingFrontCamera ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
break;
case UIDeviceOrientationLandscapeRight:
rotation = usingFrontCamera ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
break;
case UIDeviceOrientationFaceUp:
case UIDeviceOrientationFaceDown:
case UIDeviceOrientationUnknown:
// Ignore.
break;
}
#else
// No rotation on Mac.
webrtc::VideoRotation rotation = webrtc::kVideoRotation_0;
#endif
_capturer->CaptureSampleBuffer(sampleBuffer, rotation);
}
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didDropSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection {
RTCLogError(@"Dropped sample buffer.");
}
#pragma mark - AVCaptureSession notifications
- (void)handleCaptureSessionInterruption:(NSNotification *)notification {
NSString *reasonString = nil;
#if TARGET_OS_IPHONE
NSNumber *reason = notification.userInfo[AVCaptureSessionInterruptionReasonKey];
if (reason) {
switch (reason.intValue) {
case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableInBackground:
reasonString = @"VideoDeviceNotAvailableInBackground";
break;
case AVCaptureSessionInterruptionReasonAudioDeviceInUseByAnotherClient:
reasonString = @"AudioDeviceInUseByAnotherClient";
break;
case AVCaptureSessionInterruptionReasonVideoDeviceInUseByAnotherClient:
reasonString = @"VideoDeviceInUseByAnotherClient";
break;
case AVCaptureSessionInterruptionReasonVideoDeviceNotAvailableWithMultipleForegroundApps:
reasonString = @"VideoDeviceNotAvailableWithMultipleForegroundApps";
break;
}
}
#endif
RTCLog(@"Capture session interrupted: %@", reasonString);
// TODO(tkchin): Handle this case.
}
- (void)handleCaptureSessionInterruptionEnded:(NSNotification *)notification {
RTCLog(@"Capture session interruption ended.");
// TODO(tkchin): Handle this case.
}
- (void)handleCaptureSessionRuntimeError:(NSNotification *)notification {
NSError *error = [notification.userInfo objectForKey:AVCaptureSessionErrorKey];
RTCLogError(@"Capture session runtime error: %@", error);
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
#if TARGET_OS_IPHONE
if (error.code == AVErrorMediaServicesWereReset) {
[self handleNonFatalError];
} else {
[self handleFatalError];
}
#else
[self handleFatalError];
#endif
}];
}
- (void)handleCaptureSessionDidStartRunning:(NSNotification *)notification {
RTCLog(@"Capture session started.");
self.isRunning = YES;
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
// If we successfully restarted after an unknown error,
// allow future retries on fatal errors.
_hasRetriedOnFatalError = NO;
}];
}
- (void)handleCaptureSessionDidStopRunning:(NSNotification *)notification {
RTCLog(@"Capture session stopped.");
self.isRunning = NO;
}
- (void)handleFatalError {
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
if (!_hasRetriedOnFatalError) {
RTCLogWarning(@"Attempting to recover from fatal capture error.");
[self handleNonFatalError];
_hasRetriedOnFatalError = YES;
} else {
RTCLogError(@"Previous fatal error recovery failed.");
}
}];
}
- (void)handleNonFatalError {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
if (self.hasStarted) {
RTCLog(@"Restarting capture session after error.");
[self.captureSession startRunning];
}
}];
}
#if TARGET_OS_IPHONE
#pragma mark - UIApplication notifications
- (void)handleApplicationDidBecomeActive:(NSNotification *)notification {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
if (self.hasStarted && !self.captureSession.isRunning) {
RTCLog(@"Restarting capture session on active.");
[self.captureSession startRunning];
}
}];
}
#endif // TARGET_OS_IPHONE
#pragma mark - Private
- (BOOL)setupCaptureSession {
AVCaptureSession *captureSession = [[AVCaptureSession alloc] init];
#if defined(WEBRTC_IOS)
captureSession.usesApplicationAudioSession = NO;
#endif
// Add the output.
AVCaptureVideoDataOutput *videoDataOutput = [self videoDataOutput];
if (![captureSession canAddOutput:videoDataOutput]) {
RTCLogError(@"Video data output unsupported.");
return NO;
}
[captureSession addOutput:videoDataOutput];
// Get the front and back cameras. If there isn't a front camera
// give up.
AVCaptureDeviceInput *frontCameraInput = [self frontCameraInput];
AVCaptureDeviceInput *backCameraInput = [self backCameraInput];
if (!frontCameraInput) {
RTCLogError(@"No front camera for capture session.");
return NO;
}
// Add the inputs.
if (![captureSession canAddInput:frontCameraInput] ||
(backCameraInput && ![captureSession canAddInput:backCameraInput])) {
RTCLogError(@"Session does not support capture inputs.");
return NO;
}
AVCaptureDeviceInput *input = self.useBackCamera ? backCameraInput : frontCameraInput;
[captureSession addInput:input];
_captureSession = captureSession;
return YES;
}
- (AVCaptureVideoDataOutput *)videoDataOutput {
if (!_videoDataOutput) {
// Make the capturer output NV12. Ideally we want I420 but that's not
// currently supported on iPhone / iPad.
AVCaptureVideoDataOutput *videoDataOutput = [[AVCaptureVideoDataOutput alloc] init];
videoDataOutput.videoSettings = @{
(NSString *)
// TODO(denicija): Remove this color conversion and use the original capture format directly.
kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
};
videoDataOutput.alwaysDiscardsLateVideoFrames = NO;
[videoDataOutput setSampleBufferDelegate:self queue:self.frameQueue];
_videoDataOutput = videoDataOutput;
}
return _videoDataOutput;
}
- (AVCaptureDevice *)videoCaptureDeviceForPosition:(AVCaptureDevicePosition)position {
for (AVCaptureDevice *captureDevice in [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]) {
if (captureDevice.position == position) {
return captureDevice;
}
}
return nil;
}
- (AVCaptureDeviceInput *)frontCameraInput {
if (!_frontCameraInput) {
#if TARGET_OS_IPHONE
AVCaptureDevice *frontCameraDevice =
[self videoCaptureDeviceForPosition:AVCaptureDevicePositionFront];
#else
AVCaptureDevice *frontCameraDevice =
[AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
#endif
if (!frontCameraDevice) {
RTCLogWarning(@"Failed to find front capture device.");
return nil;
}
NSError *error = nil;
AVCaptureDeviceInput *frontCameraInput =
[AVCaptureDeviceInput deviceInputWithDevice:frontCameraDevice error:&error];
if (!frontCameraInput) {
RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
return nil;
}
_frontCameraInput = frontCameraInput;
}
return _frontCameraInput;
}
- (AVCaptureDeviceInput *)backCameraInput {
if (!_backCameraInput) {
AVCaptureDevice *backCameraDevice =
[self videoCaptureDeviceForPosition:AVCaptureDevicePositionBack];
if (!backCameraDevice) {
RTCLogWarning(@"Failed to find front capture device.");
return nil;
}
NSError *error = nil;
AVCaptureDeviceInput *backCameraInput =
[AVCaptureDeviceInput deviceInputWithDevice:backCameraDevice error:&error];
if (!backCameraInput) {
RTCLogError(@"Failed to create front camera input: %@", error.localizedDescription);
return nil;
}
_backCameraInput = backCameraInput;
}
return _backCameraInput;
}
// Called from capture session queue.
- (void)updateOrientation {
#if TARGET_OS_IPHONE
_orientation = [UIDevice currentDevice].orientation;
#endif
}
// Update the current session input to match what's stored in _useBackCamera.
- (void)updateSessionInputForUseBackCamera:(BOOL)useBackCamera {
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
[_captureSession beginConfiguration];
AVCaptureDeviceInput *oldInput = _backCameraInput;
AVCaptureDeviceInput *newInput = _frontCameraInput;
if (useBackCamera) {
oldInput = _frontCameraInput;
newInput = _backCameraInput;
}
if (oldInput) {
// Ok to remove this even if it's not attached. Will be no-op.
[_captureSession removeInput:oldInput];
}
if (newInput) {
[_captureSession addInput:newInput];
}
[self updateOrientation];
AVCaptureDevice *newDevice = newInput.device;
const cricket::VideoFormat *format = _capturer->GetCaptureFormat();
webrtc::SetFormatForCaptureDevice(newDevice, _captureSession, *format);
[_captureSession commitConfiguration];
}];
}
@end