sdk/objc/native/src/objc_video_encoder_factory.mm

/*
 *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "sdk/objc/native/src/objc_video_encoder_factory.h"

#include <optional>
#include <string>

#import "base/RTCVideoEncoder.h"
#import "base/RTCVideoEncoderFactory.h"
#import "components/video_codec/RTCCodecSpecificInfoH264+Private.h"
#import "sdk/objc/api/peerconnection/RTCEncodedImage+Private.h"
#import "sdk/objc/api/peerconnection/RTCVideoCodecInfo+Private.h"
#import "sdk/objc/api/peerconnection/RTCVideoEncoderSettings+Private.h"
#import "sdk/objc/api/video_codec/RTCNativeVideoEncoderBuilder+Native.h"
#import "sdk/objc/api/video_codec/RTCVideoCodecConstants.h"
#import "sdk/objc/base/RTCMacros.h"
#import "sdk/objc/helpers/NSString+StdString.h"

#include "api/environment/environment.h"
#include "api/video/video_frame.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "rtc_base/logging.h"
#include "sdk/objc/native/src/objc_video_frame.h"

namespace webrtc {

namespace {

class ObjCVideoEncoder : public VideoEncoder {
 public:
  ObjCVideoEncoder(id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder)
      : encoder_(encoder),
        implementation_name_([encoder implementationName].stdString) {}

  int32_t InitEncode(const VideoCodec *codec_settings,
                     const Settings &encoder_settings) override {
    RTC_OBJC_TYPE(RTCVideoEncoderSettings) *settings =
        [[RTC_OBJC_TYPE(RTCVideoEncoderSettings) alloc]
            initWithNativeVideoCodec:codec_settings];
    return [encoder_ startEncodeWithSettings:settings
                               numberOfCores:encoder_settings.number_of_cores];
  }

  int32_t RegisterEncodeCompleteCallback(
      EncodedImageCallback *callback) override {
    if (callback) {
      [encoder_ setCallback:^BOOL(
                    RTC_OBJC_TYPE(RTCEncodedImage) *_Nonnull frame,
                    id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)> _Nonnull info) {
        EncodedImage encodedImage = [frame nativeEncodedImage];

        // Handle types that can be converted into one of CodecSpecificInfo's
        // hard coded cases.
        CodecSpecificInfo codecSpecificInfo;
        if ([info isKindOfClass:[RTC_OBJC_TYPE(RTCCodecSpecificInfoH264)
                                    class]]) {
          codecSpecificInfo = [(RTC_OBJC_TYPE(
              RTCCodecSpecificInfoH264) *)info nativeCodecSpecificInfo];
        }

        EncodedImageCallback::Result res =
            callback->OnEncodedImage(encodedImage, &codecSpecificInfo);
        return res.error == EncodedImageCallback::Result::OK;
      }];
    } else {
      [encoder_ setCallback:nil];
    }
    return WEBRTC_VIDEO_CODEC_OK;
  }

  int32_t Release() override { return [encoder_ releaseEncoder]; }

  int32_t Encode(const VideoFrame &frame,
                 const std::vector<VideoFrameType> *frame_types) override {
    NSMutableArray<NSNumber *> *rtcFrameTypes = [NSMutableArray array];
    for (size_t i = 0; i < frame_types->size(); ++i) {
      [rtcFrameTypes addObject:@(RTCFrameType(frame_types->at(i)))];
    }

    return [encoder_ encode:ToObjCVideoFrame(frame)
          codecSpecificInfo:nil
                 frameTypes:rtcFrameTypes];
  }

  void SetRates(const RateControlParameters &parameters) override {
    const uint32_t bitrate = parameters.bitrate.get_sum_kbps();
    const uint32_t framerate =
        static_cast<uint32_t>(parameters.framerate_fps + 0.5);
    [encoder_ setBitrate:bitrate framerate:framerate];
  }

  VideoEncoder::EncoderInfo GetEncoderInfo() const override {
    EncoderInfo info;
    info.implementation_name = implementation_name_;

    RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *qp_thresholds =
        [encoder_ scalingSettings];
    info.scaling_settings = qp_thresholds ?
        ScalingSettings(qp_thresholds.low, qp_thresholds.high) :
        ScalingSettings::kOff;

    info.requested_resolution_alignment = encoder_.resolutionAlignment > 0 ?: 1;
    info.apply_alignment_to_all_simulcast_layers =
        encoder_.applyAlignmentToAllSimulcastLayers;
    info.supports_native_handle = encoder_.supportsNativeHandle;
    info.is_hardware_accelerated = true;
    return info;
  }

 private:
  id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder_;
  const std::string implementation_name_;
};

class ObjcVideoEncoderSelector
    : public VideoEncoderFactory::EncoderSelectorInterface {
 public:
  ObjcVideoEncoderSelector(
      id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector) {
    selector_ = selector;
  }
  void OnCurrentEncoder(const SdpVideoFormat &format) override {
    RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [[RTC_OBJC_TYPE(RTCVideoCodecInfo)
        alloc] initWithNativeSdpVideoFormat:format];
    [selector_ registerCurrentEncoderInfo:info];
  }
  std::optional<SdpVideoFormat> OnEncoderBroken() override {
    RTC_OBJC_TYPE(RTCVideoCodecInfo) *info =
        [selector_ encoderForBrokenEncoder];
    if (info) {
      return [info nativeSdpVideoFormat];
    }
    return std::nullopt;
  }
  std::optional<SdpVideoFormat> OnAvailableBitrate(
      const DataRate &rate) override {
    RTC_OBJC_TYPE(RTCVideoCodecInfo) *info =
        [selector_ encoderForBitrate:rate.kbps<NSInteger>()];
    if (info) {
      return [info nativeSdpVideoFormat];
    }
    return std::nullopt;
  }

  std::optional<SdpVideoFormat> OnResolutionChange(
      const RenderResolution &resolution) override {
    if ([selector_
            respondsToSelector:@selector(encoderForResolutionChangeBySize:)]) {
      RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [selector_
          encoderForResolutionChangeBySize:CGSizeMake(resolution.Width(),
                                                      resolution.Height())];
      if (info) {
        return [info nativeSdpVideoFormat];
      }
    }
    return std::nullopt;
  }

 private:
  id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector_;
};

}  // namespace

ObjCVideoEncoderFactory::ObjCVideoEncoderFactory(
    id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)> encoder_factory)
    : encoder_factory_(encoder_factory) {}

ObjCVideoEncoderFactory::~ObjCVideoEncoderFactory() {}

id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
    ObjCVideoEncoderFactory::wrapped_encoder_factory() const {
  return encoder_factory_;
}

std::vector<SdpVideoFormat> ObjCVideoEncoderFactory::GetSupportedFormats()
    const {
  std::vector<SdpVideoFormat> supported_formats;
  for (RTC_OBJC_TYPE(RTCVideoCodecInfo) *
       supportedCodec in [encoder_factory_ supportedCodecs]) {
    SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
    supported_formats.push_back(format);
  }

  return supported_formats;
}

std::vector<SdpVideoFormat> ObjCVideoEncoderFactory::GetImplementations()
    const {
  if ([encoder_factory_ respondsToSelector:@selector(implementations)]) {
    std::vector<SdpVideoFormat> supported_formats;
    for (RTC_OBJC_TYPE(RTCVideoCodecInfo) *
         supportedCodec in [encoder_factory_ implementations]) {
      SdpVideoFormat format = [supportedCodec nativeSdpVideoFormat];
      supported_formats.push_back(format);
    }
    return supported_formats;
  }
  return GetSupportedFormats();
}

VideoEncoderFactory::CodecSupport ObjCVideoEncoderFactory::QueryCodecSupport(
    const SdpVideoFormat &format,
    std::optional<std::string> scalability_mode) const {
  if ([encoder_factory_ respondsToSelector:@selector(queryCodecSupport:
                                                       scalabilityMode:)]) {
    RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [[RTC_OBJC_TYPE(RTCVideoCodecInfo)
        alloc] initWithNativeSdpVideoFormat:format];
    NSString *mode;
    if (scalability_mode.has_value()) {
      mode = [NSString stringForAbslStringView:*scalability_mode];
    }

    RTC_OBJC_TYPE(RTCVideoEncoderCodecSupport) *result =
        [encoder_factory_ queryCodecSupport:info scalabilityMode:mode];
    return {.is_supported = result.isSupported,
            .is_power_efficient = result.isPowerEfficient};
  }

  // Use default implementation.
  return VideoEncoderFactory::QueryCodecSupport(format, scalability_mode);
}

std::unique_ptr<VideoEncoder> ObjCVideoEncoderFactory::Create(
    const Environment &env, const SdpVideoFormat &format) {
  RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [[RTC_OBJC_TYPE(RTCVideoCodecInfo)
      alloc] initWithNativeSdpVideoFormat:format];
  id<RTC_OBJC_TYPE(RTCVideoEncoder)> encoder =
      [encoder_factory_ createEncoder:info];
  if ([encoder conformsToProtocol:@protocol(RTC_OBJC_TYPE(
                                      RTCNativeVideoEncoderBuilder))]) {
    return
        [((id<RTC_OBJC_TYPE(RTCNativeVideoEncoderBuilder)>)encoder) build:env];
  } else {
    return std::make_unique<ObjCVideoEncoder>(encoder);
  }
}

std::unique_ptr<VideoEncoderFactory::EncoderSelectorInterface>
    ObjCVideoEncoderFactory::GetEncoderSelector() const {
  if ([encoder_factory_ respondsToSelector:@selector(encoderSelector)]) {
    id<RTC_OBJC_TYPE(RTCVideoEncoderSelector)> selector =
        [encoder_factory_ encoderSelector];
    if (selector) {
      return absl::make_unique<ObjcVideoEncoderSelector>(selector);
    }
  }
  return nullptr;
}

}  // namespace webrtc