diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index bdf5baddcf..11dafc1c21 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -277,6 +277,31 @@ rtc_static_library("webrtc_vp9") { } if (rtc_include_tests) { + if (is_ios || is_mac) { + rtc_static_library("objc_codec_test") { + sources = [ + "codecs/test/objc_codec_h264_test.h", + "codecs/test/objc_codec_h264_test.mm", + ] + deps = [ + "../../api:video_frame_api", + "../../api/video_codecs:video_codecs_api", + "../../media:rtc_audio_video", + "../../modules:module_api", + "../../sdk:objc_common", + "../../sdk:objc_peerconnection", + "../../sdk:objc_peerconnectionfactory", + "../../sdk:objc_videotoolbox", + "../../sdk:objc_videotracksource", + ] + + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } + } + rtc_source_set("simulcast_test_utility") { testonly = true sources = [ @@ -399,6 +424,7 @@ if (rtc_include_tests) { if (is_ios || is_mac) { deps += [ + ":objc_codec_test", "../../media:rtc_media_base", "../../sdk:objc_videotoolbox", ] @@ -435,6 +461,10 @@ if (rtc_include_tests) { "../video_capture", ] + if (is_ios || is_mac) { + deps += [ ":objc_codec_test" ] + } + if (rtc_use_h264) { defines = [ "WEBRTC_USE_H264" ] } @@ -499,7 +529,10 @@ if (rtc_include_tests) { } if (is_ios || is_mac) { - deps += [ ":plot_videoprocessor_integrationtest_bundle_data" ] + deps += [ + ":objc_codec_test", + ":plot_videoprocessor_integrationtest_bundle_data", + ] } # TODO(brandtr): Remove this define when the modules_tests target properly diff --git a/webrtc/modules/video_coding/DEPS b/webrtc/modules/video_coding/DEPS index 9896c19c5d..df2a440766 100644 --- a/webrtc/modules/video_coding/DEPS +++ b/webrtc/modules/video_coding/DEPS @@ -13,7 +13,7 @@ specific_include_rules = { "+base/android", "+webrtc/sdk", ], - "(.*test\.cc|.*test\.h)": [ + "(.*test\.cc|.*test\.h|.*test\.mm)": [ "+webrtc/media/engine", "+webrtc/sdk", ], diff --git a/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h b/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h new file mode 100644 index 0000000000..0b82e927d6 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h @@ -0,0 +1,26 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_H264_TEST_H_ +#define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_H264_TEST_H_ + +#include + +#include "webrtc/media/engine/webrtcvideodecoderfactory.h" +#include "webrtc/media/engine/webrtcvideoencoderfactory.h" + +namespace webrtc { + +std::unique_ptr CreateObjCEncoderFactory(); +std::unique_ptr CreateObjCDecoderFactory(); + +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_OBJC_CODEC_H264_TEST_H_ diff --git a/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.mm b/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.mm new file mode 100644 index 0000000000..ceb661a694 --- /dev/null +++ b/webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.mm @@ -0,0 +1,29 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h" + +#import "WebRTC/RTCVideoCodecH264.h" +#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h" +#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.h" + +namespace webrtc { + +std::unique_ptr CreateObjCEncoderFactory() { + return std::unique_ptr( + new ObjCVideoEncoderFactory([[RTCVideoEncoderFactoryH264 alloc] init])); +} + +std::unique_ptr CreateObjCDecoderFactory() { + return std::unique_ptr( + new ObjCVideoDecoderFactory([[RTCVideoDecoderFactoryH264 alloc] init])); +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h index b90a85bba6..18244691fe 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.h @@ -23,8 +23,7 @@ #include "webrtc/sdk/android/src/jni/androidmediadecoder_jni.h" #include "webrtc/sdk/android/src/jni/androidmediaencoder_jni.h" #elif defined(WEBRTC_IOS) -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h" -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" +#include "webrtc/modules/video_coding/codecs/test/objc_codec_h264_test.h" #endif #include "webrtc/media/engine/webrtcvideodecoderfactory.h" @@ -178,9 +177,13 @@ class VideoProcessorIntegrationTest : public testing::Test { #elif defined(WEBRTC_IOS) ASSERT_EQ(kVideoCodecH264, config_.codec_settings->codecType) << "iOS HW codecs only support H264."; - encoder_.reset(new H264VideoToolboxEncoder( - cricket::VideoCodec(cricket::kH264CodecName))); - decoder_.reset(new H264VideoToolboxDecoder()); + std::unique_ptr encoder_factory = + CreateObjCEncoderFactory(); + std::unique_ptr decoder_factory = + CreateObjCDecoderFactory(); + cricket::VideoCodec codecInfo = encoder_factory->supported_codecs().at(0); + encoder_.reset(encoder_factory->CreateVideoEncoder(codecInfo)); + decoder_.reset(decoder_factory->CreateVideoDecoder(kVideoCodecH264)); #else RTC_NOTREACHED() << "Only support HW codecs on Android and iOS."; #endif diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index e80a2da960..3809aa112c 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -697,10 +697,10 @@ if (is_ios || is_mac) { rtc_static_library("objc_videotoolbox") { sources = [ - "objc/Framework/Classes/VideoToolbox/decoder.h", - "objc/Framework/Classes/VideoToolbox/decoder.mm", - "objc/Framework/Classes/VideoToolbox/encoder.h", - "objc/Framework/Classes/VideoToolbox/encoder.mm", + "objc/Framework/Classes/VideoToolbox/RTCVideoDecoderH264.mm", + "objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm", + "objc/Framework/Classes/VideoToolbox/helpers.cc", + "objc/Framework/Classes/VideoToolbox/helpers.h", "objc/Framework/Classes/VideoToolbox/nalu_rewriter.cc", "objc/Framework/Classes/VideoToolbox/nalu_rewriter.h", "objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.h", diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm index d386ff761f..ac57844b97 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCEncodedImage.mm @@ -27,6 +27,7 @@ @synthesize rotation = _rotation; @synthesize completeFrame = _completeFrame; @synthesize qp = _qp; +@synthesize contentType = _contentType; - (instancetype)initWithNativeEncodedImage:(webrtc::EncodedImage)encodedImage { if (self = [super init]) { @@ -46,6 +47,9 @@ _rotation = encodedImage.rotation_; _completeFrame = encodedImage._completeFrame; _qp = encodedImage.qp_ == -1 ? nil : @(encodedImage.qp_); + _contentType = (encodedImage.content_type_ == webrtc::VideoContentType::SCREENSHARE) ? + RTCVideoContentTypeScreenshare : + RTCVideoContentTypeUnspecified; } return self; @@ -67,6 +71,9 @@ encodedImage.rotation_ = webrtc::VideoRotation(_rotation); encodedImage._completeFrame = _completeFrame; encodedImage.qp_ = _qp ? _qp.intValue : -1; + encodedImage.content_type_ = (_contentType == RTCVideoContentTypeScreenshare) ? + webrtc::VideoContentType::SCREENSHARE : + webrtc::VideoContentType::UNSPECIFIED; return encodedImage; } diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h index ddd2e4e62d..269e505780 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec+Private.h @@ -29,7 +29,6 @@ NS_ASSUME_NONNULL_BEGIN @interface RTCVideoEncoderSettings () - (instancetype)initWithNativeVideoCodec:(const webrtc::VideoCodec *__nullable)videoCodec; -- (std::unique_ptr)createNativeVideoEncoderSettings; @end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm index b1fcd1b8a6..ae7215f0f1 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm @@ -19,18 +19,11 @@ #include "webrtc/rtc_base/timeutils.h" #include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h" -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" #include "webrtc/system_wrappers/include/field_trial.h" -const size_t kDefaultPayloadSize = 1440; - const char kHighProfileExperiment[] = "WebRTC-H264HighProfile"; - -// These thresholds deviate from the default h264 QP thresholds, as they have been found to work -// better on devices that support VideoToolbox -const int kLowH264QpThreshold = 28; -const int kHighH264QpThreshold = 39; +static NSString *kLevel31ConstrainedHigh = @"640c1f"; +static NSString *kLevel31ConstrainedBaseline = @"42e01f"; bool IsHighProfileEnabled() { return webrtc::field_trial::IsEnabled(kHighProfileExperiment); @@ -53,210 +46,6 @@ bool IsHighProfileEnabled() { @end -namespace { - -class H264VideoToolboxEncodeCompleteCallback : public webrtc::EncodedImageCallback { - public: - Result OnEncodedImage(const webrtc::EncodedImage &encoded_image, - const webrtc::CodecSpecificInfo *codec_specific_info, - const webrtc::RTPFragmentationHeader *fragmentation) { - RTCEncodedImage *image = [[RTCEncodedImage alloc] initWithNativeEncodedImage:encoded_image]; - - RTCCodecSpecificInfoH264 *info = [[RTCCodecSpecificInfoH264 alloc] init]; - info.packetizationMode = - (RTCH264PacketizationMode)codec_specific_info->codecSpecific.H264.packetization_mode; - - RTCRtpFragmentationHeader *header = - [[RTCRtpFragmentationHeader alloc] initWithNativeFragmentationHeader:fragmentation]; - - callback(image, info, header); - return Result(Result::OK, 0); - } - - RTCVideoEncoderCallback callback; -}; - -class H264VideoToolboxDecodeCompleteCallback : public webrtc::DecodedImageCallback { - public: - int32_t Decoded(webrtc::VideoFrame &decodedImage) { - rtc::scoped_refptr video_frame_buffer = - decodedImage.video_frame_buffer(); - id rtcFrameBuffer; - rtc::scoped_refptr objc_frame_buffer( - static_cast(video_frame_buffer.get())); - rtcFrameBuffer = (id)objc_frame_buffer->wrapped_frame_buffer(); - - RTCVideoFrame *videoFrame = [[RTCVideoFrame alloc] - initWithBuffer:rtcFrameBuffer - rotation:static_cast(decodedImage.rotation()) - timeStampNs:decodedImage.timestamp_us() * rtc::kNumNanosecsPerMicrosec]; - videoFrame.timeStamp = decodedImage.timestamp(); - - callback(videoFrame); - - return 0; - } - - RTCVideoDecoderCallback callback; -}; - -} // namespace - -// Encoder. -@implementation RTCVideoEncoderH264 { - webrtc::H264VideoToolboxEncoder *_videoToolboxEncoder; - H264VideoToolboxEncodeCompleteCallback *_toolboxCallback; -} - -- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo { - if (self = [super init]) { - cricket::VideoCodec codec = [codecInfo nativeVideoCodec]; - _videoToolboxEncoder = new webrtc::H264VideoToolboxEncoder(codec); - } - return self; -} - -- (void)setCallback:(RTCVideoEncoderCallback)callback { - if (!_toolboxCallback) _toolboxCallback = new H264VideoToolboxEncodeCompleteCallback(); - _toolboxCallback->callback = callback; - _videoToolboxEncoder->RegisterEncodeCompleteCallback(_toolboxCallback); -} - -- (void)destroy { - delete _videoToolboxEncoder; - _videoToolboxEncoder = nullptr; - delete _toolboxCallback; - _toolboxCallback = nullptr; -} - --(void)dealloc { - // Make sure C++ objects have been properly cleaned up before this - // is dealloc'd. - RTC_DCHECK(!_videoToolboxEncoder); - RTC_DCHECK(!_toolboxCallback); -} - -- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings - numberOfCores:(int)numberOfCores { - std::unique_ptr codecSettings = [settings createNativeVideoEncoderSettings]; - return _videoToolboxEncoder->InitEncode( - codecSettings.release(), numberOfCores, kDefaultPayloadSize); -} - -- (NSInteger)releaseEncoder { - return _videoToolboxEncoder->Release(); -} - -- (NSInteger)encode:(RTCVideoFrame *)frame - codecSpecificInfo:(id)info - frameTypes:(NSArray *)frameTypes { - rtc::scoped_refptr frameBuffer = - new rtc::RefCountedObject(frame.buffer); - webrtc::VideoFrame videoFrame(frameBuffer, - (webrtc::VideoRotation)frame.rotation, - frame.timeStampNs / rtc::kNumNanosecsPerMicrosec); - videoFrame.set_timestamp(frame.timeStamp); - - // Handle types than can be converted into one of webrtc::CodecSpecificInfo's hard coded cases. - webrtc::CodecSpecificInfo codecSpecificInfo; - if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) { - codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info nativeCodecSpecificInfo]; - } - - std::vector nativeFrameTypes; - for (NSNumber *frameType in frameTypes) { - RTCFrameType rtcFrameType = (RTCFrameType)frameType.unsignedIntegerValue; - nativeFrameTypes.push_back((webrtc::FrameType)rtcFrameType); - } - - return _videoToolboxEncoder->Encode(videoFrame, &codecSpecificInfo, &nativeFrameTypes); -} - -- (BOOL)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { - return _videoToolboxEncoder->SetRates(bitrateKbit, framerate) == WEBRTC_VIDEO_CODEC_OK; -} - -- (RTCVideoEncoderQpThresholds *)scalingSettings { - return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold - high:kHighH264QpThreshold]; -} - -- (NSString *)implementationName { - return @"VideoToolbox"; -} - -@end - -// Decoder. -@implementation RTCVideoDecoderH264 { - webrtc::H264VideoToolboxDecoder *_videoToolboxDecoder; - H264VideoToolboxDecodeCompleteCallback *_toolboxCallback; -} - -- (instancetype)init { - if (self = [super init]) { - cricket::VideoCodec codec(cricket::kH264CodecName); - _videoToolboxDecoder = new webrtc::H264VideoToolboxDecoder(); - } - return self; -} - -- (NSInteger)startDecodeWithSettings:(RTCVideoEncoderSettings *)settings - numberOfCores:(int)numberOfCores { - std::unique_ptr codecSettings = [settings createNativeVideoEncoderSettings]; - return _videoToolboxDecoder->InitDecode(codecSettings.release(), numberOfCores); -} - -- (void)setCallback:(RTCVideoDecoderCallback)callback { - if (!_toolboxCallback) _toolboxCallback = new H264VideoToolboxDecodeCompleteCallback(); - _toolboxCallback->callback = callback; - _videoToolboxDecoder->RegisterDecodeCompleteCallback(_toolboxCallback); -} - -- (NSInteger)releaseDecoder { - return _videoToolboxDecoder->Release(); -} - -- (void)destroy { - delete _videoToolboxDecoder; - _videoToolboxDecoder = nullptr; - delete _toolboxCallback; - _toolboxCallback = nullptr; -} - --(void)dealloc { - // Make sure C++ objects have been properly cleaned up before this - // is dealloc'd. - RTC_DCHECK(!_videoToolboxDecoder); - RTC_DCHECK(!_toolboxCallback); -} - -- (NSInteger)decode:(RTCEncodedImage *)encodedImage - missingFrames:(BOOL)missingFrames - fragmentationHeader:(RTCRtpFragmentationHeader *)fragmentationHeader - codecSpecificInfo:(__nullable id)info - renderTimeMs:(int64_t)renderTimeMs { - webrtc::EncodedImage image = [encodedImage nativeEncodedImage]; - - // Handle types than can be converted into one of webrtc::CodecSpecificInfo's hard coded cases. - webrtc::CodecSpecificInfo codecSpecificInfo; - if ([info isKindOfClass:[RTCCodecSpecificInfoH264 class]]) { - codecSpecificInfo = [(RTCCodecSpecificInfoH264 *)info nativeCodecSpecificInfo]; - } - - std::unique_ptr header = - [fragmentationHeader createNativeFragmentationHeader]; - - return _videoToolboxDecoder->Decode( - image, missingFrames, header.release(), &codecSpecificInfo, renderTimeMs); -} - -- (NSString *)implementationName { - return @"VideoToolbox"; -} - -@end - // Encoder factory. @implementation RTCVideoEncoderFactoryH264 @@ -266,7 +55,7 @@ class H264VideoToolboxDecodeCompleteCallback : public webrtc::DecodedImageCallba if (IsHighProfileEnabled()) { NSDictionary *constrainedHighParams = @{ - @"profile-level-id" : @"640c1f", // Level 3.1 Constrained High. + @"profile-level-id" : kLevel31ConstrainedHigh, @"level-asymmetry-allowed" : @"1", @"packetization-mode" : @"1", }; @@ -278,7 +67,7 @@ class H264VideoToolboxDecodeCompleteCallback : public webrtc::DecodedImageCallba } NSDictionary *constrainedBaselineParams = @{ - @"profile-level-id" : @"42e01f", // Level 3.1 Constrained Baseline. + @"profile-level-id" : kLevel31ConstrainedBaseline, @"level-asymmetry-allowed" : @"1", @"packetization-mode" : @"1", }; diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoEncoderSettings.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoEncoderSettings.mm index 018e76a74e..af82e0c4f2 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoEncoderSettings.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoEncoderSettings.mm @@ -25,6 +25,7 @@ @synthesize targetBitrate = _targetBitrate; @synthesize maxFramerate = _maxFramerate; @synthesize qpMax = _qpMax; +@synthesize mode = _mode; - (instancetype)initWithNativeVideoCodec:(const webrtc::VideoCodec *)videoCodec { if (self = [super init]) { @@ -42,31 +43,11 @@ _targetBitrate = videoCodec->targetBitrate; _maxFramerate = videoCodec->maxFramerate; _qpMax = videoCodec->qpMax; + _mode = (RTCVideoCodecMode)videoCodec->mode; } } return self; } -- (std::unique_ptr)createNativeVideoEncoderSettings { - auto codecSettings = std::unique_ptr(new webrtc::VideoCodec); - - rtc::Optional codecType = - webrtc::PayloadNameToCodecType([NSString stdStringForString:_name]); - if (codecType) { - codecSettings->codecType = codecType.value(); - } - - codecSettings->width = _width; - codecSettings->height = _height; - codecSettings->startBitrate = _startBitrate; - codecSettings->maxBitrate = _maxBitrate; - codecSettings->minBitrate = _minBitrate; - codecSettings->targetBitrate = _targetBitrate; - codecSettings->maxFramerate = _maxFramerate; - codecSettings->qpMax = _qpMax; - - return codecSettings; -} - @end diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoDecoderH264.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoDecoderH264.mm new file mode 100644 index 0000000000..f657f9c2f1 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoDecoderH264.mm @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "WebRTC/RTCVideoCodecH264.h" + +#import + +#include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/checks.h" +#include "webrtc/rtc_base/logging.h" +#include "webrtc/rtc_base/timeutils.h" +#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" + +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" +#import "helpers.h" + +#if defined(WEBRTC_IOS) +#import "Common/RTCUIApplicationStatusObserver.h" +#endif + +// Struct that we pass to the decoder per frame to decode. We receive it again +// in the decoder callback. +struct RTCFrameDecodeParams { + RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) : callback(cb), timestamp(ts) {} + RTCVideoDecoderCallback callback; + int64_t timestamp; +}; + +// This is the callback function that VideoToolbox calls when decode is +// complete. +void decompressionOutputCallback(void *decoder, + void *params, + OSStatus status, + VTDecodeInfoFlags infoFlags, + CVImageBufferRef imageBuffer, + CMTime timestamp, + CMTime duration) { + std::unique_ptr decodeParams( + reinterpret_cast(params)); + if (status != noErr) { + LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; + return; + } + // TODO(tkchin): Handle CVO properly. + RTCCVPixelBuffer *frameBuffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:imageBuffer]; + RTCVideoFrame *decodedFrame = + [[RTCVideoFrame alloc] initWithBuffer:frameBuffer + rotation:RTCVideoRotation_0 + timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec]; + decodedFrame.timeStamp = decodeParams->timestamp; + decodeParams->callback(decodedFrame); +} + +// Decoder. +@implementation RTCVideoDecoderH264 { + CMVideoFormatDescriptionRef _videoFormat; + VTDecompressionSessionRef _decompressionSession; + RTCVideoDecoderCallback _callback; +} + +- (void)dealloc { + [self destroyDecompressionSession]; + [self setVideoFormat:nullptr]; +} + +- (NSInteger)startDecodeWithSettings:(RTCVideoEncoderSettings *)settings + numberOfCores:(int)numberOfCores { + return WEBRTC_VIDEO_CODEC_OK; +} + +- (NSInteger)decode:(RTCEncodedImage *)inputImage + missingFrames:(BOOL)missingFrames + fragmentationHeader:(RTCRtpFragmentationHeader *)fragmentationHeader + codecSpecificInfo:(__nullable id)info + renderTimeMs:(int64_t)renderTimeMs { + RTC_DCHECK(inputImage.buffer); + +#if defined(WEBRTC_IOS) + if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { + // Ignore all decode requests when app isn't active. In this state, the + // hardware decoder has been invalidated by the OS. + // Reset video format so that we won't process frames until the next + // keyframe. + [self setVideoFormat:nullptr]; + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } +#endif + CMVideoFormatDescriptionRef inputFormat = nullptr; + if (webrtc::H264AnnexBBufferHasVideoFormatDescription((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length)) { + inputFormat = webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length); + if (inputFormat) { + // Check if the video format has changed, and reinitialize decoder if + // needed. + if (!CMFormatDescriptionEqual(inputFormat, _videoFormat)) { + [self setVideoFormat:inputFormat]; + [self resetDecompressionSession]; + } + CFRelease(inputFormat); + } + } + if (!_videoFormat) { + // We received a frame but we don't have format information so we can't + // decode it. + // This can happen after backgrounding. We need to wait for the next + // sps/pps before we can resume so we request a keyframe by returning an + // error. + LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + CMSampleBufferRef sampleBuffer = nullptr; + if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length, + _videoFormat, + &sampleBuffer)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(sampleBuffer); + VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression; + std::unique_ptr frameDecodeParams; + frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp)); + OSStatus status = VTDecompressionSessionDecodeFrame( + _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); +#if defined(WEBRTC_IOS) + // Re-initialize the decoder if we have an invalid session while the app is + // active and retry the decode request. + if (status == kVTInvalidSessionErr && [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) { + frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp)); + status = VTDecompressionSessionDecodeFrame( + _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); + } +#endif + CFRelease(sampleBuffer); + if (status != noErr) { + LOG(LS_ERROR) << "Failed to decode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)setCallback:(RTCVideoDecoderCallback)callback { + _callback = callback; +} + +- (NSInteger)releaseDecoder { + // Need to invalidate the session so that callbacks no longer occur and it + // is safe to null out the callback. + [self destroyDecompressionSession]; + [self setVideoFormat:nullptr]; + _callback = nullptr; + return WEBRTC_VIDEO_CODEC_OK; +} + +#pragma mark - Private + +- (int)resetDecompressionSession { + [self destroyDecompressionSession]; + + // Need to wait for the first SPS to initialize decoder. + if (!_videoFormat) { + return WEBRTC_VIDEO_CODEC_OK; + } + + // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder + // create pixel buffers with GPU backed memory. The intent here is to pass + // the pixel buffers directly so we avoid a texture upload later during + // rendering. This currently is moot because we are converting back to an + // I420 frame after decode, but eventually we will be able to plumb + // CVPixelBuffers directly to the renderer. + // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that + // we can pass CVPixelBuffers as native handles in decoder output. + static size_t const attributesSize = 3; + CFTypeRef keys[attributesSize] = { +#if defined(WEBRTC_IOS) + kCVPixelBufferOpenGLESCompatibilityKey, +#elif defined(WEBRTC_MAC) + kCVPixelBufferOpenGLCompatibilityKey, +#endif + kCVPixelBufferIOSurfacePropertiesKey, + kCVPixelBufferPixelFormatTypeKey + }; + CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); + int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); + CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat}; + CFDictionaryRef attributes = CreateCFTypeDictionary(keys, values, attributesSize); + if (ioSurfaceValue) { + CFRelease(ioSurfaceValue); + ioSurfaceValue = nullptr; + } + if (pixelFormat) { + CFRelease(pixelFormat); + pixelFormat = nullptr; + } + VTDecompressionOutputCallbackRecord record = { + decompressionOutputCallback, nullptr, + }; + OSStatus status = VTDecompressionSessionCreate( + nullptr, _videoFormat, nullptr, attributes, &record, &_decompressionSession); + CFRelease(attributes); + if (status != noErr) { + [self destroyDecompressionSession]; + return WEBRTC_VIDEO_CODEC_ERROR; + } + [self configureDecompressionSession]; + + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)configureDecompressionSession { + RTC_DCHECK(_decompressionSession); +#if defined(WEBRTC_IOS) + VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); +#endif +} + +- (void)destroyDecompressionSession { + if (_decompressionSession) { + VTDecompressionSessionInvalidate(_decompressionSession); + CFRelease(_decompressionSession); + _decompressionSession = nullptr; + } +} + +- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat { + if (_videoFormat == videoFormat) { + return; + } + if (_videoFormat) { + CFRelease(_videoFormat); + } + _videoFormat = videoFormat; + if (_videoFormat) { + CFRetain(_videoFormat); + } +} + +- (NSString *)implementationName { + return @"VideoToolbox"; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm new file mode 100644 index 0000000000..4091886dd4 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/RTCVideoEncoderH264.mm @@ -0,0 +1,680 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#import "WebRTC/RTCVideoCodecH264.h" + +#import +#include + +#if defined(WEBRTC_IOS) +#import "Common/RTCUIApplicationStatusObserver.h" +#import "WebRTC/UIDevice+RTCDevice.h" +#endif +#import "PeerConnection/RTCVideoCodec+Private.h" +#import "WebRTC/RTCVideoCodec.h" +#import "WebRTC/RTCVideoFrame.h" +#import "WebRTC/RTCVideoFrameBuffer.h" +#import "helpers.h" +#include "libyuv/convert_from.h" +#include "webrtc/common_video/h264/h264_bitstream_parser.h" +#include "webrtc/common_video/h264/profile_level_id.h" +#include "webrtc/common_video/include/bitrate_adjuster.h" +#include "webrtc/modules/include/module_common_types.h" +#include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/rtc_base/buffer.h" +#include "webrtc/rtc_base/logging.h" +#include "webrtc/rtc_base/timeutils.h" +#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" +#include "webrtc/system_wrappers/include/clock.h" + +@interface RTCVideoEncoderH264 () + +- (void)frameWasEncoded:(OSStatus)status + flags:(VTEncodeInfoFlags)infoFlags + sampleBuffer:(CMSampleBufferRef)sampleBuffer + codecSpecificInfo:(id)codecSpecificInfo + width:(int32_t)width + height:(int32_t)height + renderTimeMs:(int64_t)renderTimeMs + timestamp:(uint32_t)timestamp + rotation:(RTCVideoRotation)rotation; + +@end + +// The ratio between kVTCompressionPropertyKey_DataRateLimits and +// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher +// than the average bit rate to avoid undershooting the target. +const float kLimitToAverageBitRateFactor = 1.5f; +// These thresholds deviate from the default h264 QP thresholds, as they +// have been found to work better on devices that support VideoToolbox +const int kLowH264QpThreshold = 28; +const int kHighH264QpThreshold = 39; + +// Struct that we pass to the encoder per frame to encode. We receive it again +// in the encoder callback. +struct RTCFrameEncodeParams { + RTCFrameEncodeParams(RTCVideoEncoderH264 *e, + RTCCodecSpecificInfoH264 *csi, + int32_t w, + int32_t h, + int64_t rtms, + uint32_t ts, + RTCVideoRotation r) + : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) { + if (csi) { + codecSpecificInfo = csi; + } else { + codecSpecificInfo = [[RTCCodecSpecificInfoH264 alloc] init]; + } + } + + RTCVideoEncoderH264 *encoder; + RTCCodecSpecificInfoH264 *codecSpecificInfo; + int32_t width; + int32_t height; + int64_t render_time_ms; + uint32_t timestamp; + RTCVideoRotation rotation; +}; + +// We receive I420Frames as input, but we need to feed CVPixelBuffers into the +// encoder. This performs the copy and format conversion. +// TODO(tkchin): See if encoder will accept i420 frames and compare performance. +bool CopyVideoFrameToPixelBuffer(id frameBuffer, CVPixelBufferRef pixelBuffer) { + RTC_DCHECK(pixelBuffer); + RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); + RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), frameBuffer.height); + RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), frameBuffer.width); + + CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (cvRet != kCVReturnSuccess) { + LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; + return false; + } + uint8_t *dstY = reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); + int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); + uint8_t *dstUV = reinterpret_cast(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); + int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); + // Convert I420 to NV12. + int ret = libyuv::I420ToNV12(frameBuffer.dataY, + frameBuffer.strideY, + frameBuffer.dataU, + frameBuffer.strideU, + frameBuffer.dataV, + frameBuffer.strideV, + dstY, + dstStrideY, + dstUV, + dstStrideUV, + frameBuffer.width, + frameBuffer.height); + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + if (ret) { + LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; + return false; + } + return true; +} + +CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { + if (!pixel_buffer_pool) { + LOG(LS_ERROR) << "Failed to get pixel buffer pool."; + return nullptr; + } + CVPixelBufferRef pixel_buffer; + CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer); + if (ret != kCVReturnSuccess) { + LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; + // We probably want to drop frames here, since failure probably means + // that the pool is empty. + return nullptr; + } + return pixel_buffer; +} + +// This is the callback function that VideoToolbox calls when encode is +// complete. From inspection this happens on its own queue. +void compressionOutputCallback(void *encoder, + void *params, + OSStatus status, + VTEncodeInfoFlags infoFlags, + CMSampleBufferRef sampleBuffer) { + std::unique_ptr encodeParams( + reinterpret_cast(params)); + [encodeParams->encoder frameWasEncoded:status + flags:infoFlags + sampleBuffer:sampleBuffer + codecSpecificInfo:encodeParams->codecSpecificInfo + width:encodeParams->width + height:encodeParams->height + renderTimeMs:encodeParams->render_time_ms + timestamp:encodeParams->timestamp + rotation:encodeParams->rotation]; +} + +// Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no +// specific VideoToolbox profile for the specified level, AutoLevel will be +// returned. The user must initialize the encoder with a resolution and +// framerate conforming to the selected H264 level regardless. +CFStringRef ExtractProfile(const cricket::VideoCodec &codec) { + const rtc::Optional profile_level_id = + webrtc::H264::ParseSdpProfileLevelId(codec.params); + RTC_DCHECK(profile_level_id); + switch (profile_level_id->profile) { + case webrtc::H264::kProfileConstrainedBaseline: + case webrtc::H264::kProfileBaseline: + switch (profile_level_id->level) { + case webrtc::H264::kLevel3: + return kVTProfileLevel_H264_Baseline_3_0; + case webrtc::H264::kLevel3_1: + return kVTProfileLevel_H264_Baseline_3_1; + case webrtc::H264::kLevel3_2: + return kVTProfileLevel_H264_Baseline_3_2; + case webrtc::H264::kLevel4: + return kVTProfileLevel_H264_Baseline_4_0; + case webrtc::H264::kLevel4_1: + return kVTProfileLevel_H264_Baseline_4_1; + case webrtc::H264::kLevel4_2: + return kVTProfileLevel_H264_Baseline_4_2; + case webrtc::H264::kLevel5: + return kVTProfileLevel_H264_Baseline_5_0; + case webrtc::H264::kLevel5_1: + return kVTProfileLevel_H264_Baseline_5_1; + case webrtc::H264::kLevel5_2: + return kVTProfileLevel_H264_Baseline_5_2; + case webrtc::H264::kLevel1: + case webrtc::H264::kLevel1_b: + case webrtc::H264::kLevel1_1: + case webrtc::H264::kLevel1_2: + case webrtc::H264::kLevel1_3: + case webrtc::H264::kLevel2: + case webrtc::H264::kLevel2_1: + case webrtc::H264::kLevel2_2: + return kVTProfileLevel_H264_Baseline_AutoLevel; + } + + case webrtc::H264::kProfileMain: + switch (profile_level_id->level) { + case webrtc::H264::kLevel3: + return kVTProfileLevel_H264_Main_3_0; + case webrtc::H264::kLevel3_1: + return kVTProfileLevel_H264_Main_3_1; + case webrtc::H264::kLevel3_2: + return kVTProfileLevel_H264_Main_3_2; + case webrtc::H264::kLevel4: + return kVTProfileLevel_H264_Main_4_0; + case webrtc::H264::kLevel4_1: + return kVTProfileLevel_H264_Main_4_1; + case webrtc::H264::kLevel4_2: + return kVTProfileLevel_H264_Main_4_2; + case webrtc::H264::kLevel5: + return kVTProfileLevel_H264_Main_5_0; + case webrtc::H264::kLevel5_1: + return kVTProfileLevel_H264_Main_5_1; + case webrtc::H264::kLevel5_2: + return kVTProfileLevel_H264_Main_5_2; + case webrtc::H264::kLevel1: + case webrtc::H264::kLevel1_b: + case webrtc::H264::kLevel1_1: + case webrtc::H264::kLevel1_2: + case webrtc::H264::kLevel1_3: + case webrtc::H264::kLevel2: + case webrtc::H264::kLevel2_1: + case webrtc::H264::kLevel2_2: + return kVTProfileLevel_H264_Main_AutoLevel; + } + + case webrtc::H264::kProfileConstrainedHigh: + case webrtc::H264::kProfileHigh: + switch (profile_level_id->level) { + case webrtc::H264::kLevel3: + return kVTProfileLevel_H264_High_3_0; + case webrtc::H264::kLevel3_1: + return kVTProfileLevel_H264_High_3_1; + case webrtc::H264::kLevel3_2: + return kVTProfileLevel_H264_High_3_2; + case webrtc::H264::kLevel4: + return kVTProfileLevel_H264_High_4_0; + case webrtc::H264::kLevel4_1: + return kVTProfileLevel_H264_High_4_1; + case webrtc::H264::kLevel4_2: + return kVTProfileLevel_H264_High_4_2; + case webrtc::H264::kLevel5: + return kVTProfileLevel_H264_High_5_0; + case webrtc::H264::kLevel5_1: + return kVTProfileLevel_H264_High_5_1; + case webrtc::H264::kLevel5_2: + return kVTProfileLevel_H264_High_5_2; + case webrtc::H264::kLevel1: + case webrtc::H264::kLevel1_b: + case webrtc::H264::kLevel1_1: + case webrtc::H264::kLevel1_2: + case webrtc::H264::kLevel1_3: + case webrtc::H264::kLevel2: + case webrtc::H264::kLevel2_1: + case webrtc::H264::kLevel2_2: + return kVTProfileLevel_H264_High_AutoLevel; + } + } +} + +@implementation RTCVideoEncoderH264 { + RTCVideoCodecInfo *_codecInfo; + webrtc::BitrateAdjuster *_bitrateAdjuster; + uint32_t _targetBitrateBps; + uint32_t _encoderBitrateBps; + RTCH264PacketizationMode _packetizationMode; + CFStringRef _profile; + RTCVideoEncoderCallback _callback; + int32_t _width; + int32_t _height; + VTCompressionSessionRef _compressionSession; + RTCVideoCodecMode _mode; + + webrtc::H264BitstreamParser _h264BitstreamParser; + std::vector _nv12ScaleBuffer; +} + +// .5 is set as a mininum to prevent overcompensating for large temporary +// overshoots. We don't want to degrade video quality too badly. +// .95 is set to prevent oscillations. When a lower bitrate is set on the +// encoder than previously set, its output seems to have a brief period of +// drastically reduced bitrate, so we want to avoid that. In steady state +// conditions, 0.95 seems to give us better overall bitrate over long periods +// of time. +- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo { + if (self = [super init]) { + _codecInfo = codecInfo; + _bitrateAdjuster = new webrtc::BitrateAdjuster(webrtc::Clock::GetRealTimeClock(), .5, .95); + _packetizationMode = RTCH264PacketizationModeNonInterleaved; + _profile = ExtractProfile([codecInfo nativeVideoCodec]); + LOG(LS_INFO) << "Using profile " << CFStringToString(_profile); + RTC_CHECK([codecInfo.name isEqualToString:@"H264"]); + } + return self; +} + +- (void)dealloc { + [self destroyCompressionSession]; +} + +- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings + numberOfCores:(int)numberOfCores { + RTC_DCHECK(settings); + RTC_DCHECK([settings.name isEqualToString:@"H264"]); + + _width = settings.width; + _height = settings.height; + _mode = settings.mode; + + // We can only set average bitrate on the HW encoder. + _targetBitrateBps = settings.startBitrate; + _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + + // TODO(tkchin): Try setting payload size via + // kVTCompressionPropertyKey_MaxH264SliceBytes. + + return [self resetCompressionSession]; +} + +- (NSInteger)encode:(RTCVideoFrame *)frame + codecSpecificInfo:(id)codecSpecificInfo + frameTypes:(NSArray *)frameTypes { + RTC_DCHECK_EQ(frame.width, _width); + RTC_DCHECK_EQ(frame.height, _height); + if (!_callback || !_compressionSession) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } +#if defined(WEBRTC_IOS) + if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { + // Ignore all encode requests when app isn't active. In this state, the + // hardware encoder has been invalidated by the OS. + return WEBRTC_VIDEO_CODEC_OK; + } +#endif + BOOL isKeyframeRequired = NO; + + // Get a pixel buffer from the pool and copy frame data over. + CVPixelBufferPoolRef pixelBufferPool = + VTCompressionSessionGetPixelBufferPool(_compressionSession); + +#if defined(WEBRTC_IOS) + if (!pixelBufferPool) { + // Kind of a hack. On backgrounding, the compression session seems to get + // invalidated, which causes this pool call to fail when the application + // is foregrounded and frames are being sent for encoding again. + // Resetting the session when this happens fixes the issue. + // In addition we request a keyframe so video can recover quickly. + [self resetCompressionSession]; + pixelBufferPool = VTCompressionSessionGetPixelBufferPool(_compressionSession); + isKeyframeRequired = YES; + LOG(LS_INFO) << "Resetting compression session due to invalid pool."; + } +#endif + + CVPixelBufferRef pixelBuffer = nullptr; + if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { + // Native frame buffer + RTCCVPixelBuffer *rtcPixelBuffer = (RTCCVPixelBuffer *)frame.buffer; + if (![rtcPixelBuffer requiresCropping]) { + // This pixel buffer might have a higher resolution than what the + // compression session is configured to. The compression session can + // handle that and will output encoded frames in the configured + // resolution regardless of the input pixel buffer resolution. + pixelBuffer = rtcPixelBuffer.pixelBuffer; + CVBufferRetain(pixelBuffer); + } else { + // Cropping required, we need to crop and scale to a new pixel buffer. + pixelBuffer = CreatePixelBuffer(pixelBufferPool); + if (!pixelBuffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + int dstWidth = CVPixelBufferGetWidth(pixelBuffer); + int dstHeight = CVPixelBufferGetHeight(pixelBuffer); + if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) { + int size = + [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth height:dstHeight]; + _nv12ScaleBuffer.resize(size); + } else { + _nv12ScaleBuffer.clear(); + } + _nv12ScaleBuffer.shrink_to_fit(); + if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer withTempBuffer:_nv12ScaleBuffer.data()]) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + } + + if (!pixelBuffer) { + // We did not have a native frame buffer + pixelBuffer = CreatePixelBuffer(pixelBufferPool); + if (!pixelBuffer) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + RTC_DCHECK(pixelBuffer); + if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) { + LOG(LS_ERROR) << "Failed to copy frame data."; + CVBufferRelease(pixelBuffer); + return WEBRTC_VIDEO_CODEC_ERROR; + } + } + + // Check if we need a keyframe. + if (!isKeyframeRequired && frameTypes) { + for (NSNumber *frameType in frameTypes) { + if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) { + isKeyframeRequired = YES; + break; + } + } + } + + CMTime presentationTimeStamp = CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); + CFDictionaryRef frameProperties = nullptr; + if (isKeyframeRequired) { + CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; + CFTypeRef values[] = {kCFBooleanTrue}; + frameProperties = CreateCFTypeDictionary(keys, values, 1); + } + + std::unique_ptr encodeParams; + encodeParams.reset(new RTCFrameEncodeParams(self, + codecSpecificInfo, + _width, + _height, + frame.timeStampNs / rtc::kNumNanosecsPerMillisec, + frame.timeStamp, + frame.rotation)); + encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode; + + // Update the bitrate if needed. + [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; + + OSStatus status = VTCompressionSessionEncodeFrame(_compressionSession, + pixelBuffer, + presentationTimeStamp, + kCMTimeInvalid, + frameProperties, + encodeParams.release(), + nullptr); + if (frameProperties) { + CFRelease(frameProperties); + } + if (pixelBuffer) { + CVBufferRelease(pixelBuffer); + } + if (status != noErr) { + LOG(LS_ERROR) << "Failed to encode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)setCallback:(RTCVideoEncoderCallback)callback { + _callback = callback; +} + +- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { + _targetBitrateBps = 1000 * bitrateKbit; + _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); + [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; + return WEBRTC_VIDEO_CODEC_OK; +} + +#pragma mark - Private + +- (NSInteger)releaseEncoder { + // Need to destroy so that the session is invalidated and won't use the + // callback anymore. Do not remove callback until the session is invalidated + // since async encoder callbacks can occur until invalidation. + [self destroyCompressionSession]; + _callback = nullptr; + return WEBRTC_VIDEO_CODEC_OK; +} + +- (int)resetCompressionSession { + [self destroyCompressionSession]; + + // Set source image buffer attributes. These attributes will be present on + // buffers retrieved from the encoder's pixel buffer pool. + const size_t attributesSize = 3; + CFTypeRef keys[attributesSize] = { +#if defined(WEBRTC_IOS) + kCVPixelBufferOpenGLESCompatibilityKey, +#elif defined(WEBRTC_MAC) + kCVPixelBufferOpenGLCompatibilityKey, +#endif + kCVPixelBufferIOSurfacePropertiesKey, + kCVPixelBufferPixelFormatTypeKey + }; + CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); + int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; + CFNumberRef pixelFormat = CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); + CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, pixelFormat}; + CFDictionaryRef sourceAttributes = CreateCFTypeDictionary(keys, values, attributesSize); + if (ioSurfaceValue) { + CFRelease(ioSurfaceValue); + ioSurfaceValue = nullptr; + } + if (pixelFormat) { + CFRelease(pixelFormat); + pixelFormat = nullptr; + } + OSStatus status = VTCompressionSessionCreate(nullptr, // use default allocator + _width, + _height, + kCMVideoCodecType_H264, + nullptr, // use default encoder + sourceAttributes, + nullptr, // use default compressed data allocator + compressionOutputCallback, + nullptr, + &_compressionSession); + if (sourceAttributes) { + CFRelease(sourceAttributes); + sourceAttributes = nullptr; + } + if (status != noErr) { + LOG(LS_ERROR) << "Failed to create compression session: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } + [self configureCompressionSession]; + return WEBRTC_VIDEO_CODEC_OK; +} + +- (void)configureCompressionSession { + RTC_DCHECK(_compressionSession); + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true); + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, _profile); + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false); + [self setEncoderBitrateBps:_targetBitrateBps]; + // TODO(tkchin): Look at entropy mode and colorspace matrices. + // TODO(tkchin): Investigate to see if there's any way to make this work. + // May need it to interop with Android. Currently this call just fails. + // On inspecting encoder output on iOS8, this value is set to 6. + // internal::SetVTSessionProperty(compression_session_, + // kVTCompressionPropertyKey_MaxFrameDelayCount, + // 1); + + // Set a relatively large value for keyframe emission (7200 frames or 4 minutes). + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); + SetVTSessionProperty( + _compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); +} + +- (void)destroyCompressionSession { + if (_compressionSession) { + VTCompressionSessionInvalidate(_compressionSession); + CFRelease(_compressionSession); + _compressionSession = nullptr; + } +} + +- (NSString *)implementationName { + return @"VideoToolbox"; +} + +- (void)setBitrateBps:(uint32_t)bitrateBps { + if (_encoderBitrateBps != bitrateBps) { + [self setEncoderBitrateBps:bitrateBps]; + } +} + +- (void)setEncoderBitrateBps:(uint32_t)bitrateBps { + if (_compressionSession) { + SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps); + + // TODO(tkchin): Add a helper method to set array value. + int64_t dataLimitBytesPerSecondValue = + static_cast(bitrateBps * kLimitToAverageBitRateFactor / 8); + CFNumberRef bytesPerSecond = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue); + int64_t oneSecondValue = 1; + CFNumberRef oneSecond = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue); + const void *nums[2] = {bytesPerSecond, oneSecond}; + CFArrayRef dataRateLimits = CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); + OSStatus status = VTSessionSetProperty( + _compressionSession, kVTCompressionPropertyKey_DataRateLimits, dataRateLimits); + if (bytesPerSecond) { + CFRelease(bytesPerSecond); + } + if (oneSecond) { + CFRelease(oneSecond); + } + if (dataRateLimits) { + CFRelease(dataRateLimits); + } + if (status != noErr) { + LOG(LS_ERROR) << "Failed to set data rate limit"; + } + + _encoderBitrateBps = bitrateBps; + } +} + +- (void)frameWasEncoded:(OSStatus)status + flags:(VTEncodeInfoFlags)infoFlags + sampleBuffer:(CMSampleBufferRef)sampleBuffer + codecSpecificInfo:(id)codecSpecificInfo + width:(int32_t)width + height:(int32_t)height + renderTimeMs:(int64_t)renderTimeMs + timestamp:(uint32_t)timestamp + rotation:(RTCVideoRotation)rotation { + if (status != noErr) { + LOG(LS_ERROR) << "H264 encode failed."; + return; + } + if (infoFlags & kVTEncodeInfo_FrameDropped) { + LOG(LS_INFO) << "H264 encode dropped frame."; + return; + } + + BOOL isKeyframe = NO; + CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); + if (attachments != nullptr && CFArrayGetCount(attachments)) { + CFDictionaryRef attachment = + static_cast(CFArrayGetValueAtIndex(attachments, 0)); + isKeyframe = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); + } + + if (isKeyframe) { + LOG(LS_INFO) << "Generated keyframe"; + } + + // Convert the sample buffer into a buffer suitable for RTP packetization. + // TODO(tkchin): Allocate buffers through a pool. + std::unique_ptr buffer(new rtc::Buffer()); + RTCRtpFragmentationHeader *header; + { + webrtc::RTPFragmentationHeader *header_cpp; + bool result = + H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get(), &header_cpp); + header = [[RTCRtpFragmentationHeader alloc] initWithNativeFragmentationHeader:header_cpp]; + if (!result) { + return; + } + } + + RTCEncodedImage *frame = [[RTCEncodedImage alloc] init]; + frame.buffer = [NSData dataWithBytesNoCopy:buffer->data() length:buffer->size() freeWhenDone:NO]; + frame.encodedWidth = width; + frame.encodedHeight = height; + frame.completeFrame = YES; + frame.frameType = isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta; + frame.captureTimeMs = renderTimeMs; + frame.timeStamp = timestamp; + frame.rotation = rotation; + frame.contentType = (_mode == RTCVideoCodecModeScreensharing) ? RTCVideoContentTypeScreenshare : + RTCVideoContentTypeUnspecified; + frame.isTimingFrame = NO; + + int qp; + _h264BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); + _h264BitstreamParser.GetLastSliceQp(&qp); + frame.qp = @(qp); + + BOOL res = _callback(frame, codecSpecificInfo, header); + if (!res) { + LOG(LS_ERROR) << "Encode callback failed"; + return; + } + _bitrateAdjuster->Update(frame.buffer.length); +} + +- (RTCVideoEncoderQpThresholds *)scalingSettings { + return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold + high:kHighH264QpThreshold]; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h deleted file mode 100644 index f2252da806..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_DECODER_H_ -#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_DECODER_H_ - -#include "webrtc/modules/video_coding/codecs/h264/include/h264.h" - -#include - -// This file provides a H264 encoder implementation using the VideoToolbox -// APIs. Since documentation is almost non-existent, this is largely based on -// the information in the VideoToolbox header files, a talk from WWDC 2014 and -// experimentation. - -namespace webrtc { - -class H264VideoToolboxDecoder : public H264Decoder { - public: - H264VideoToolboxDecoder(); - - ~H264VideoToolboxDecoder() override; - - int InitDecode(const VideoCodec* video_codec, int number_of_cores) override; - - int Decode(const EncodedImage& input_image, - bool missing_frames, - const RTPFragmentationHeader* fragmentation, - const CodecSpecificInfo* codec_specific_info, - int64_t render_time_ms) override; - - int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override; - - int Release() override; - - const char* ImplementationName() const override; - - private: - int ResetDecompressionSession(); - void ConfigureDecompressionSession(); - void DestroyDecompressionSession(); - void SetVideoFormat(CMVideoFormatDescriptionRef video_format); - - DecodedImageCallback* callback_; - CMVideoFormatDescriptionRef video_format_; - VTDecompressionSessionRef decompression_session_; -}; // H264VideoToolboxDecoder - -} // namespace webrtc - -#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_DECODER_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm deleted file mode 100644 index a8da8091d9..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.mm +++ /dev/null @@ -1,277 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/decoder.h" - -#include - -#include "libyuv/convert.h" -#include "webrtc/api/video/video_frame.h" -#include "webrtc/common_video/include/video_frame.h" -#include "webrtc/rtc_base/checks.h" -#include "webrtc/rtc_base/logging.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" - -#import "WebRTC/RTCVideoFrameBuffer.h" - -#if defined(WEBRTC_IOS) -#import "Common/RTCUIApplicationStatusObserver.h" -#endif - -namespace webrtc { -namespace { - -static const int64_t kMsPerSec = 1000; - -// Convenience function for creating a dictionary. -inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, - CFTypeRef* values, - size_t size) { - return CFDictionaryCreate(nullptr, keys, values, size, - &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks); -} - -// Struct that we pass to the decoder per frame to decode. We receive it again -// in the decoder callback. -struct FrameDecodeParams { - FrameDecodeParams(DecodedImageCallback* cb, int64_t ts) - : callback(cb), timestamp(ts) {} - DecodedImageCallback* callback; - int64_t timestamp; -}; - -// This is the callback function that VideoToolbox calls when decode is -// complete. -void VTDecompressionOutputCallback(void* decoder, - void* params, - OSStatus status, - VTDecodeInfoFlags info_flags, - CVImageBufferRef image_buffer, - CMTime timestamp, - CMTime duration) { - std::unique_ptr decode_params( - reinterpret_cast(params)); - if (status != noErr) { - LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; - return; - } - // TODO(tkchin): Handle CVO properly. - rtc::scoped_refptr buffer = new rtc::RefCountedObject( - [[RTCCVPixelBuffer alloc] initWithPixelBuffer:image_buffer]); - VideoFrame decoded_frame(buffer, decode_params->timestamp, - CMTimeGetSeconds(timestamp) * kMsPerSec, - kVideoRotation_0); - decode_params->callback->Decoded(decoded_frame); -} - -} // namespace - -H264VideoToolboxDecoder::H264VideoToolboxDecoder() - : callback_(nullptr), video_format_(nullptr), decompression_session_(nullptr) {} - -H264VideoToolboxDecoder::~H264VideoToolboxDecoder() { - DestroyDecompressionSession(); - SetVideoFormat(nullptr); -} - -int H264VideoToolboxDecoder::InitDecode(const VideoCodec* video_codec, - int number_of_cores) { - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxDecoder::Decode( - const EncodedImage& input_image, - bool missing_frames, - const RTPFragmentationHeader* fragmentation, - const CodecSpecificInfo* codec_specific_info, - int64_t render_time_ms) { - RTC_DCHECK(input_image._buffer); - -#if defined(WEBRTC_IOS) - if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { - // Ignore all decode requests when app isn't active. In this state, the - // hardware decoder has been invalidated by the OS. - // Reset video format so that we won't process frames until the next - // keyframe. - SetVideoFormat(nullptr); - return WEBRTC_VIDEO_CODEC_NO_OUTPUT; - } -#endif - CMVideoFormatDescriptionRef input_format = nullptr; - if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, - input_image._length)) { - input_format = CreateVideoFormatDescription(input_image._buffer, - input_image._length); - if (input_format) { - // Check if the video format has changed, and reinitialize decoder if - // needed. - if (!CMFormatDescriptionEqual(input_format, video_format_)) { - SetVideoFormat(input_format); - ResetDecompressionSession(); - } - CFRelease(input_format); - } - } - if (!video_format_) { - // We received a frame but we don't have format information so we can't - // decode it. - // This can happen after backgrounding. We need to wait for the next - // sps/pps before we can resume so we request a keyframe by returning an - // error. - LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; - return WEBRTC_VIDEO_CODEC_ERROR; - } - CMSampleBufferRef sample_buffer = nullptr; - if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, - input_image._length, video_format_, - &sample_buffer)) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_DCHECK(sample_buffer); - VTDecodeFrameFlags decode_flags = - kVTDecodeFrame_EnableAsynchronousDecompression; - std::unique_ptr frame_decode_params; - frame_decode_params.reset( - new FrameDecodeParams(callback_, input_image._timeStamp)); - OSStatus status = VTDecompressionSessionDecodeFrame( - decompression_session_, sample_buffer, decode_flags, - frame_decode_params.release(), nullptr); -#if defined(WEBRTC_IOS) - // Re-initialize the decoder if we have an invalid session while the app is - // active and retry the decode request. - if (status == kVTInvalidSessionErr && - ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { - frame_decode_params.reset( - new FrameDecodeParams(callback_, input_image._timeStamp)); - status = VTDecompressionSessionDecodeFrame( - decompression_session_, sample_buffer, decode_flags, - frame_decode_params.release(), nullptr); - } -#endif - CFRelease(sample_buffer); - if (status != noErr) { - LOG(LS_ERROR) << "Failed to decode frame with code: " << status; - return WEBRTC_VIDEO_CODEC_ERROR; - } - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxDecoder::RegisterDecodeCompleteCallback( - DecodedImageCallback* callback) { - RTC_DCHECK(!callback_); - callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxDecoder::Release() { - // Need to invalidate the session so that callbacks no longer occur and it - // is safe to null out the callback. - DestroyDecompressionSession(); - SetVideoFormat(nullptr); - callback_ = nullptr; - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxDecoder::ResetDecompressionSession() { - DestroyDecompressionSession(); - - // Need to wait for the first SPS to initialize decoder. - if (!video_format_) { - return WEBRTC_VIDEO_CODEC_OK; - } - - // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder - // create pixel buffers with GPU backed memory. The intent here is to pass - // the pixel buffers directly so we avoid a texture upload later during - // rendering. This currently is moot because we are converting back to an - // I420 frame after decode, but eventually we will be able to plumb - // CVPixelBuffers directly to the renderer. - // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that - // we can pass CVPixelBuffers as native handles in decoder output. - static size_t const attributes_size = 3; - CFTypeRef keys[attributes_size] = { -#if defined(WEBRTC_IOS) - kCVPixelBufferOpenGLESCompatibilityKey, -#elif defined(WEBRTC_MAC) - kCVPixelBufferOpenGLCompatibilityKey, -#endif - kCVPixelBufferIOSurfacePropertiesKey, - kCVPixelBufferPixelFormatTypeKey - }; - CFDictionaryRef io_surface_value = CreateCFDictionary(nullptr, nullptr, 0); - int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; - CFNumberRef pixel_format = - CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); - CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, - pixel_format}; - CFDictionaryRef attributes = - CreateCFDictionary(keys, values, attributes_size); - if (io_surface_value) { - CFRelease(io_surface_value); - io_surface_value = nullptr; - } - if (pixel_format) { - CFRelease(pixel_format); - pixel_format = nullptr; - } - VTDecompressionOutputCallbackRecord record = { - VTDecompressionOutputCallback, this, - }; - OSStatus status = - VTDecompressionSessionCreate(nullptr, video_format_, nullptr, attributes, - &record, &decompression_session_); - CFRelease(attributes); - if (status != noErr) { - DestroyDecompressionSession(); - return WEBRTC_VIDEO_CODEC_ERROR; - } - ConfigureDecompressionSession(); - - return WEBRTC_VIDEO_CODEC_OK; -} - -void H264VideoToolboxDecoder::ConfigureDecompressionSession() { - RTC_DCHECK(decompression_session_); -#if defined(WEBRTC_IOS) - VTSessionSetProperty(decompression_session_, - kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); -#endif -} - -void H264VideoToolboxDecoder::DestroyDecompressionSession() { - if (decompression_session_) { - VTDecompressionSessionInvalidate(decompression_session_); - CFRelease(decompression_session_); - decompression_session_ = nullptr; - } -} - -void H264VideoToolboxDecoder::SetVideoFormat( - CMVideoFormatDescriptionRef video_format) { - if (video_format_ == video_format) { - return; - } - if (video_format_) { - CFRelease(video_format_); - } - video_format_ = video_format; - if (video_format_) { - CFRetain(video_format_); - } -} - -const char* H264VideoToolboxDecoder::ImplementationName() const { - return "VideoToolbox"; -} - -} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h deleted file mode 100644 index c9a45551a9..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_ENCODER_H_ -#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_ENCODER_H_ - -#include "webrtc/api/video/video_rotation.h" -#include "webrtc/common_video/h264/h264_bitstream_parser.h" -#include "webrtc/common_video/include/bitrate_adjuster.h" -#include "webrtc/media/base/codec.h" -#include "webrtc/modules/video_coding/codecs/h264/include/h264.h" -#include "webrtc/modules/video_coding/utility/quality_scaler.h" -#include "webrtc/rtc_base/criticalsection.h" - -#include -#include - -// This file provides a H264 encoder implementation using the VideoToolbox -// APIs. Since documentation is almost non-existent, this is largely based on -// the information in the VideoToolbox header files, a talk from WWDC 2014 and -// experimentation. - -namespace webrtc { - -class H264VideoToolboxEncoder : public H264Encoder { - public: - explicit H264VideoToolboxEncoder(const cricket::VideoCodec& codec); - - ~H264VideoToolboxEncoder() override; - - int InitEncode(const VideoCodec* codec_settings, - int number_of_cores, - size_t max_payload_size) override; - - int Encode(const VideoFrame& input_image, - const CodecSpecificInfo* codec_specific_info, - const std::vector* frame_types) override; - - int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; - - int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - - int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override; - - int Release() override; - - const char* ImplementationName() const override; - - bool SupportsNativeHandle() const override; - - void OnEncodedFrame(OSStatus status, - VTEncodeInfoFlags info_flags, - CMSampleBufferRef sample_buffer, - CodecSpecificInfo codec_specific_info, - int32_t width, - int32_t height, - int64_t render_time_ms, - uint32_t timestamp, - VideoRotation rotation); - - ScalingSettings GetScalingSettings() const override; - - private: - int ResetCompressionSession(); - void ConfigureCompressionSession(); - void DestroyCompressionSession(); - rtc::scoped_refptr GetScaledBufferOnEncode( - const rtc::scoped_refptr& frame); - void SetBitrateBps(uint32_t bitrate_bps); - void SetEncoderBitrateBps(uint32_t bitrate_bps); - - EncodedImageCallback* callback_; - VTCompressionSessionRef compression_session_; - BitrateAdjuster bitrate_adjuster_; - H264PacketizationMode packetization_mode_; - uint32_t target_bitrate_bps_; - uint32_t encoder_bitrate_bps_; - int32_t width_; - int32_t height_; - VideoCodecMode mode_; - const CFStringRef profile_; - - H264BitstreamParser h264_bitstream_parser_; - std::vector nv12_scale_buffer_; -}; // H264VideoToolboxEncoder - -} // namespace webrtc - -#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_ENCODER_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm deleted file mode 100644 index 2bac68ef6d..0000000000 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm +++ /dev/null @@ -1,767 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - * - */ - -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.h" - -#include -#include -#include - -#if defined(WEBRTC_IOS) -#import "Common/RTCUIApplicationStatusObserver.h" -#import "WebRTC/UIDevice+RTCDevice.h" -#endif -#import "WebRTC/RTCVideoFrameBuffer.h" -#include "libyuv/convert_from.h" -#include "webrtc/common_video/h264/profile_level_id.h" -#include "webrtc/rtc_base/checks.h" -#include "webrtc/rtc_base/logging.h" -#include "webrtc/sdk/objc/Framework/Classes/Video/objc_frame_buffer.h" -#include "webrtc/sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" -#include "webrtc/system_wrappers/include/clock.h" - -namespace internal { - -// The ratio between kVTCompressionPropertyKey_DataRateLimits and -// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher -// than the average bit rate to avoid undershooting the target. -const float kLimitToAverageBitRateFactor = 1.5f; -// These thresholds deviate from the default h264 QP thresholds, as they -// have been found to work better on devices that support VideoToolbox -const int kLowH264QpThreshold = 28; -const int kHighH264QpThreshold = 39; - -// Convenience function for creating a dictionary. -inline CFDictionaryRef CreateCFDictionary(CFTypeRef* keys, - CFTypeRef* values, - size_t size) { - return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, - &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks); -} - -// Copies characters from a CFStringRef into a std::string. -std::string CFStringToString(const CFStringRef cf_string) { - RTC_DCHECK(cf_string); - std::string std_string; - // Get the size needed for UTF8 plus terminating character. - size_t buffer_size = - CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), - kCFStringEncodingUTF8) + - 1; - std::unique_ptr buffer(new char[buffer_size]); - if (CFStringGetCString(cf_string, buffer.get(), buffer_size, - kCFStringEncodingUTF8)) { - // Copy over the characters. - std_string.assign(buffer.get()); - } - return std_string; -} - -// Convenience function for setting a VT property. -void SetVTSessionProperty(VTSessionRef session, - CFStringRef key, - int32_t value) { - CFNumberRef cfNum = - CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); - OSStatus status = VTSessionSetProperty(session, key, cfNum); - CFRelease(cfNum); - if (status != noErr) { - std::string key_string = CFStringToString(key); - LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string - << " to " << value << ": " << status; - } -} - -// Convenience function for setting a VT property. -void SetVTSessionProperty(VTSessionRef session, - CFStringRef key, - uint32_t value) { - int64_t value_64 = value; - CFNumberRef cfNum = - CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); - OSStatus status = VTSessionSetProperty(session, key, cfNum); - CFRelease(cfNum); - if (status != noErr) { - std::string key_string = CFStringToString(key); - LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string - << " to " << value << ": " << status; - } -} - -// Convenience function for setting a VT property. -void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { - CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; - OSStatus status = VTSessionSetProperty(session, key, cf_bool); - if (status != noErr) { - std::string key_string = CFStringToString(key); - LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string - << " to " << value << ": " << status; - } -} - -// Convenience function for setting a VT property. -void SetVTSessionProperty(VTSessionRef session, - CFStringRef key, - CFStringRef value) { - OSStatus status = VTSessionSetProperty(session, key, value); - if (status != noErr) { - std::string key_string = CFStringToString(key); - std::string val_string = CFStringToString(value); - LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string - << " to " << val_string << ": " << status; - } -} - -// Struct that we pass to the encoder per frame to encode. We receive it again -// in the encoder callback. -struct FrameEncodeParams { - FrameEncodeParams(webrtc::H264VideoToolboxEncoder* e, - const webrtc::CodecSpecificInfo* csi, - int32_t w, - int32_t h, - int64_t rtms, - uint32_t ts, - webrtc::VideoRotation r) - : encoder(e), - width(w), - height(h), - render_time_ms(rtms), - timestamp(ts), - rotation(r) { - if (csi) { - codec_specific_info = *csi; - } else { - codec_specific_info.codecType = webrtc::kVideoCodecH264; - } - } - - webrtc::H264VideoToolboxEncoder* encoder; - webrtc::CodecSpecificInfo codec_specific_info; - int32_t width; - int32_t height; - int64_t render_time_ms; - uint32_t timestamp; - webrtc::VideoRotation rotation; -}; - -// We receive I420Frames as input, but we need to feed CVPixelBuffers into the -// encoder. This performs the copy and format conversion. -// TODO(tkchin): See if encoder will accept i420 frames and compare performance. -bool CopyVideoFrameToPixelBuffer(const rtc::scoped_refptr& frame, - CVPixelBufferRef pixel_buffer) { - RTC_DCHECK(pixel_buffer); - RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixel_buffer), - kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); - RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixel_buffer, 0), - static_cast(frame->height())); - RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixel_buffer, 0), - static_cast(frame->width())); - - CVReturn cvRet = CVPixelBufferLockBaseAddress(pixel_buffer, 0); - if (cvRet != kCVReturnSuccess) { - LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; - return false; - } - uint8_t* dst_y = reinterpret_cast( - CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); - int dst_stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); - uint8_t* dst_uv = reinterpret_cast( - CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); - int dst_stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); - // Convert I420 to NV12. - int ret = libyuv::I420ToNV12( - frame->DataY(), frame->StrideY(), - frame->DataU(), frame->StrideU(), - frame->DataV(), frame->StrideV(), - dst_y, dst_stride_y, dst_uv, dst_stride_uv, - frame->width(), frame->height()); - CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); - if (ret) { - LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; - return false; - } - return true; -} - -CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { - if (!pixel_buffer_pool) { - LOG(LS_ERROR) << "Failed to get pixel buffer pool."; - return nullptr; - } - CVPixelBufferRef pixel_buffer; - CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, - &pixel_buffer); - if (ret != kCVReturnSuccess) { - LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; - // We probably want to drop frames here, since failure probably means - // that the pool is empty. - return nullptr; - } - return pixel_buffer; -} - -// This is the callback function that VideoToolbox calls when encode is -// complete. From inspection this happens on its own queue. -void VTCompressionOutputCallback(void* encoder, - void* params, - OSStatus status, - VTEncodeInfoFlags info_flags, - CMSampleBufferRef sample_buffer) { - std::unique_ptr encode_params( - reinterpret_cast(params)); - encode_params->encoder->OnEncodedFrame( - status, info_flags, sample_buffer, encode_params->codec_specific_info, - encode_params->width, encode_params->height, - encode_params->render_time_ms, encode_params->timestamp, - encode_params->rotation); -} - -// Extract VideoToolbox profile out of the cricket::VideoCodec. If there is no -// specific VideoToolbox profile for the specified level, AutoLevel will be -// returned. The user must initialize the encoder with a resolution and -// framerate conforming to the selected H264 level regardless. -CFStringRef ExtractProfile(const cricket::VideoCodec& codec) { - const rtc::Optional profile_level_id = - webrtc::H264::ParseSdpProfileLevelId(codec.params); - RTC_DCHECK(profile_level_id); - switch (profile_level_id->profile) { - case webrtc::H264::kProfileConstrainedBaseline: - case webrtc::H264::kProfileBaseline: - switch (profile_level_id->level) { - case webrtc::H264::kLevel3: - return kVTProfileLevel_H264_Baseline_3_0; - case webrtc::H264::kLevel3_1: - return kVTProfileLevel_H264_Baseline_3_1; - case webrtc::H264::kLevel3_2: - return kVTProfileLevel_H264_Baseline_3_2; - case webrtc::H264::kLevel4: - return kVTProfileLevel_H264_Baseline_4_0; - case webrtc::H264::kLevel4_1: - return kVTProfileLevel_H264_Baseline_4_1; - case webrtc::H264::kLevel4_2: - return kVTProfileLevel_H264_Baseline_4_2; - case webrtc::H264::kLevel5: - return kVTProfileLevel_H264_Baseline_5_0; - case webrtc::H264::kLevel5_1: - return kVTProfileLevel_H264_Baseline_5_1; - case webrtc::H264::kLevel5_2: - return kVTProfileLevel_H264_Baseline_5_2; - case webrtc::H264::kLevel1: - case webrtc::H264::kLevel1_b: - case webrtc::H264::kLevel1_1: - case webrtc::H264::kLevel1_2: - case webrtc::H264::kLevel1_3: - case webrtc::H264::kLevel2: - case webrtc::H264::kLevel2_1: - case webrtc::H264::kLevel2_2: - return kVTProfileLevel_H264_Baseline_AutoLevel; - } - - case webrtc::H264::kProfileMain: - switch (profile_level_id->level) { - case webrtc::H264::kLevel3: - return kVTProfileLevel_H264_Main_3_0; - case webrtc::H264::kLevel3_1: - return kVTProfileLevel_H264_Main_3_1; - case webrtc::H264::kLevel3_2: - return kVTProfileLevel_H264_Main_3_2; - case webrtc::H264::kLevel4: - return kVTProfileLevel_H264_Main_4_0; - case webrtc::H264::kLevel4_1: - return kVTProfileLevel_H264_Main_4_1; - case webrtc::H264::kLevel4_2: - return kVTProfileLevel_H264_Main_4_2; - case webrtc::H264::kLevel5: - return kVTProfileLevel_H264_Main_5_0; - case webrtc::H264::kLevel5_1: - return kVTProfileLevel_H264_Main_5_1; - case webrtc::H264::kLevel5_2: - return kVTProfileLevel_H264_Main_5_2; - case webrtc::H264::kLevel1: - case webrtc::H264::kLevel1_b: - case webrtc::H264::kLevel1_1: - case webrtc::H264::kLevel1_2: - case webrtc::H264::kLevel1_3: - case webrtc::H264::kLevel2: - case webrtc::H264::kLevel2_1: - case webrtc::H264::kLevel2_2: - return kVTProfileLevel_H264_Main_AutoLevel; - } - - case webrtc::H264::kProfileConstrainedHigh: - case webrtc::H264::kProfileHigh: - switch (profile_level_id->level) { - case webrtc::H264::kLevel3: - return kVTProfileLevel_H264_High_3_0; - case webrtc::H264::kLevel3_1: - return kVTProfileLevel_H264_High_3_1; - case webrtc::H264::kLevel3_2: - return kVTProfileLevel_H264_High_3_2; - case webrtc::H264::kLevel4: - return kVTProfileLevel_H264_High_4_0; - case webrtc::H264::kLevel4_1: - return kVTProfileLevel_H264_High_4_1; - case webrtc::H264::kLevel4_2: - return kVTProfileLevel_H264_High_4_2; - case webrtc::H264::kLevel5: - return kVTProfileLevel_H264_High_5_0; - case webrtc::H264::kLevel5_1: - return kVTProfileLevel_H264_High_5_1; - case webrtc::H264::kLevel5_2: - return kVTProfileLevel_H264_High_5_2; - case webrtc::H264::kLevel1: - case webrtc::H264::kLevel1_b: - case webrtc::H264::kLevel1_1: - case webrtc::H264::kLevel1_2: - case webrtc::H264::kLevel1_3: - case webrtc::H264::kLevel2: - case webrtc::H264::kLevel2_1: - case webrtc::H264::kLevel2_2: - return kVTProfileLevel_H264_High_AutoLevel; - } - } -} - -} // namespace internal - -namespace webrtc { - -// .5 is set as a mininum to prevent overcompensating for large temporary -// overshoots. We don't want to degrade video quality too badly. -// .95 is set to prevent oscillations. When a lower bitrate is set on the -// encoder than previously set, its output seems to have a brief period of -// drastically reduced bitrate, so we want to avoid that. In steady state -// conditions, 0.95 seems to give us better overall bitrate over long periods -// of time. -H264VideoToolboxEncoder::H264VideoToolboxEncoder(const cricket::VideoCodec& codec) - : callback_(nullptr), - compression_session_(nullptr), - bitrate_adjuster_(Clock::GetRealTimeClock(), .5, .95), - packetization_mode_(H264PacketizationMode::NonInterleaved), - profile_(internal::ExtractProfile(codec)) { - LOG(LS_INFO) << "Using profile " << internal::CFStringToString(profile_); - RTC_CHECK(cricket::CodecNamesEq(codec.name, cricket::kH264CodecName)); -} - -H264VideoToolboxEncoder::~H264VideoToolboxEncoder() { - DestroyCompressionSession(); -} - -int H264VideoToolboxEncoder::InitEncode(const VideoCodec* codec_settings, - int number_of_cores, - size_t max_payload_size) { - RTC_DCHECK(codec_settings); - RTC_DCHECK_EQ(codec_settings->codecType, kVideoCodecH264); - - width_ = codec_settings->width; - height_ = codec_settings->height; - mode_ = codec_settings->mode; - // We can only set average bitrate on the HW encoder. - target_bitrate_bps_ = codec_settings->startBitrate; - bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); - - // TODO(tkchin): Try setting payload size via - // kVTCompressionPropertyKey_MaxH264SliceBytes. - - return ResetCompressionSession(); -} - -int H264VideoToolboxEncoder::Encode( - const VideoFrame& frame, - const CodecSpecificInfo* codec_specific_info, - const std::vector* frame_types) { - // |input_frame| size should always match codec settings. - RTC_DCHECK_EQ(frame.width(), width_); - RTC_DCHECK_EQ(frame.height(), height_); - if (!callback_ || !compression_session_) { - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } -#if defined(WEBRTC_IOS) - if (![[RTCUIApplicationStatusObserver sharedInstance] isApplicationActive]) { - // Ignore all encode requests when app isn't active. In this state, the - // hardware encoder has been invalidated by the OS. - return WEBRTC_VIDEO_CODEC_OK; - } -#endif - bool is_keyframe_required = false; - - // Get a pixel buffer from the pool and copy frame data over. - CVPixelBufferPoolRef pixel_buffer_pool = - VTCompressionSessionGetPixelBufferPool(compression_session_); -#if defined(WEBRTC_IOS) - if (!pixel_buffer_pool) { - // Kind of a hack. On backgrounding, the compression session seems to get - // invalidated, which causes this pool call to fail when the application - // is foregrounded and frames are being sent for encoding again. - // Resetting the session when this happens fixes the issue. - // In addition we request a keyframe so video can recover quickly. - ResetCompressionSession(); - pixel_buffer_pool = - VTCompressionSessionGetPixelBufferPool(compression_session_); - is_keyframe_required = true; - LOG(LS_INFO) << "Resetting compression session due to invalid pool."; - } -#endif - - CVPixelBufferRef pixel_buffer = nullptr; - if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative) { - // Native frame. - rtc::scoped_refptr objc_frame_buffer( - static_cast(frame.video_frame_buffer().get())); - id wrapped_frame_buffer = - (id)objc_frame_buffer->wrapped_frame_buffer(); - - if ([wrapped_frame_buffer isKindOfClass:[RTCCVPixelBuffer class]]) { - RTCCVPixelBuffer* rtc_pixel_buffer = (RTCCVPixelBuffer*)wrapped_frame_buffer; - if (![rtc_pixel_buffer requiresCropping]) { - // This pixel buffer might have a higher resolution than what the - // compression session is configured to. The compression session can - // handle that and will output encoded frames in the configured - // resolution regardless of the input pixel buffer resolution. - pixel_buffer = rtc_pixel_buffer.pixelBuffer; - CVBufferRetain(pixel_buffer); - } else { - // Cropping required, we need to crop and scale to a new pixel buffer. - pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); - if (!pixel_buffer) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - int dst_width = CVPixelBufferGetWidth(pixel_buffer); - int dst_height = CVPixelBufferGetHeight(pixel_buffer); - if ([rtc_pixel_buffer requiresScalingToWidth:dst_width height:dst_height]) { - int size = - [rtc_pixel_buffer bufferSizeForCroppingAndScalingToWidth:dst_width height:dst_height]; - nv12_scale_buffer_.resize(size); - } else { - nv12_scale_buffer_.clear(); - } - nv12_scale_buffer_.shrink_to_fit(); - if (![rtc_pixel_buffer cropAndScaleTo:pixel_buffer - withTempBuffer:nv12_scale_buffer_.data()]) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - } - } - } - - if (!pixel_buffer) { - // We did not have a native frame, or the ObjCVideoFrame wrapped a non-native frame - pixel_buffer = internal::CreatePixelBuffer(pixel_buffer_pool); - if (!pixel_buffer) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_DCHECK(pixel_buffer); - if (!internal::CopyVideoFrameToPixelBuffer(frame.video_frame_buffer()->ToI420(), - pixel_buffer)) { - LOG(LS_ERROR) << "Failed to copy frame data."; - CVBufferRelease(pixel_buffer); - return WEBRTC_VIDEO_CODEC_ERROR; - } - } - - // Check if we need a keyframe. - if (!is_keyframe_required && frame_types) { - for (auto frame_type : *frame_types) { - if (frame_type == kVideoFrameKey) { - is_keyframe_required = true; - break; - } - } - } - - CMTime presentation_time_stamp = - CMTimeMake(frame.render_time_ms(), 1000); - CFDictionaryRef frame_properties = nullptr; - if (is_keyframe_required) { - CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; - CFTypeRef values[] = {kCFBooleanTrue}; - frame_properties = internal::CreateCFDictionary(keys, values, 1); - } - std::unique_ptr encode_params; - encode_params.reset(new internal::FrameEncodeParams( - this, codec_specific_info, width_, height_, frame.render_time_ms(), - frame.timestamp(), frame.rotation())); - - encode_params->codec_specific_info.codecSpecific.H264.packetization_mode = - packetization_mode_; - - // Update the bitrate if needed. - SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); - - OSStatus status = VTCompressionSessionEncodeFrame( - compression_session_, pixel_buffer, presentation_time_stamp, - kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); - if (frame_properties) { - CFRelease(frame_properties); - } - if (pixel_buffer) { - CVBufferRelease(pixel_buffer); - } - if (status != noErr) { - LOG(LS_ERROR) << "Failed to encode frame with code: " << status; - return WEBRTC_VIDEO_CODEC_ERROR; - } - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxEncoder::RegisterEncodeCompleteCallback( - EncodedImageCallback* callback) { - callback_ = callback; - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxEncoder::SetChannelParameters(uint32_t packet_loss, - int64_t rtt) { - // Encoder doesn't know anything about packet loss or rtt so just return. - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxEncoder::SetRates(uint32_t new_bitrate_kbit, - uint32_t frame_rate) { - target_bitrate_bps_ = 1000 * new_bitrate_kbit; - bitrate_adjuster_.SetTargetBitrateBps(target_bitrate_bps_); - SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxEncoder::Release() { - // Need to destroy so that the session is invalidated and won't use the - // callback anymore. Do not remove callback until the session is invalidated - // since async encoder callbacks can occur until invalidation. - DestroyCompressionSession(); - callback_ = nullptr; - return WEBRTC_VIDEO_CODEC_OK; -} - -int H264VideoToolboxEncoder::ResetCompressionSession() { - DestroyCompressionSession(); - - // Set source image buffer attributes. These attributes will be present on - // buffers retrieved from the encoder's pixel buffer pool. - const size_t attributes_size = 3; - CFTypeRef keys[attributes_size] = { -#if defined(WEBRTC_IOS) - kCVPixelBufferOpenGLESCompatibilityKey, -#elif defined(WEBRTC_MAC) - kCVPixelBufferOpenGLCompatibilityKey, -#endif - kCVPixelBufferIOSurfacePropertiesKey, - kCVPixelBufferPixelFormatTypeKey - }; - CFDictionaryRef io_surface_value = - internal::CreateCFDictionary(nullptr, nullptr, 0); - int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; - CFNumberRef pixel_format = - CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); - CFTypeRef values[attributes_size] = {kCFBooleanTrue, io_surface_value, - pixel_format}; - CFDictionaryRef source_attributes = - internal::CreateCFDictionary(keys, values, attributes_size); - if (io_surface_value) { - CFRelease(io_surface_value); - io_surface_value = nullptr; - } - if (pixel_format) { - CFRelease(pixel_format); - pixel_format = nullptr; - } - OSStatus status = VTCompressionSessionCreate( - nullptr, // use default allocator - width_, height_, kCMVideoCodecType_H264, - nullptr, // use default encoder - source_attributes, - nullptr, // use default compressed data allocator - internal::VTCompressionOutputCallback, this, &compression_session_); - if (source_attributes) { - CFRelease(source_attributes); - source_attributes = nullptr; - } - if (status != noErr) { - LOG(LS_ERROR) << "Failed to create compression session: " << status; - return WEBRTC_VIDEO_CODEC_ERROR; - } - ConfigureCompressionSession(); - return WEBRTC_VIDEO_CODEC_OK; -} - -void H264VideoToolboxEncoder::ConfigureCompressionSession() { - RTC_DCHECK(compression_session_); - internal::SetVTSessionProperty(compression_session_, - kVTCompressionPropertyKey_RealTime, true); - internal::SetVTSessionProperty(compression_session_, - kVTCompressionPropertyKey_ProfileLevel, - profile_); - internal::SetVTSessionProperty(compression_session_, - kVTCompressionPropertyKey_AllowFrameReordering, - false); - SetEncoderBitrateBps(target_bitrate_bps_); - // TODO(tkchin): Look at entropy mode and colorspace matrices. - // TODO(tkchin): Investigate to see if there's any way to make this work. - // May need it to interop with Android. Currently this call just fails. - // On inspecting encoder output on iOS8, this value is set to 6. - // internal::SetVTSessionProperty(compression_session_, - // kVTCompressionPropertyKey_MaxFrameDelayCount, - // 1); - - // Set a relatively large value for keyframe emission (7200 frames or - // 4 minutes). - internal::SetVTSessionProperty( - compression_session_, - kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); - internal::SetVTSessionProperty( - compression_session_, - kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, 240); -} - -void H264VideoToolboxEncoder::DestroyCompressionSession() { - if (compression_session_) { - VTCompressionSessionInvalidate(compression_session_); - CFRelease(compression_session_); - compression_session_ = nullptr; - } -} - -const char* H264VideoToolboxEncoder::ImplementationName() const { - return "VideoToolbox"; -} - -bool H264VideoToolboxEncoder::SupportsNativeHandle() const { - return true; -} - -void H264VideoToolboxEncoder::SetBitrateBps(uint32_t bitrate_bps) { - if (encoder_bitrate_bps_ != bitrate_bps) { - SetEncoderBitrateBps(bitrate_bps); - } -} - -void H264VideoToolboxEncoder::SetEncoderBitrateBps(uint32_t bitrate_bps) { - if (compression_session_) { - internal::SetVTSessionProperty(compression_session_, - kVTCompressionPropertyKey_AverageBitRate, - bitrate_bps); - - // TODO(tkchin): Add a helper method to set array value. - int64_t data_limit_bytes_per_second_value = static_cast( - bitrate_bps * internal::kLimitToAverageBitRateFactor / 8); - CFNumberRef bytes_per_second = - CFNumberCreate(kCFAllocatorDefault, - kCFNumberSInt64Type, - &data_limit_bytes_per_second_value); - int64_t one_second_value = 1; - CFNumberRef one_second = - CFNumberCreate(kCFAllocatorDefault, - kCFNumberSInt64Type, - &one_second_value); - const void* nums[2] = { bytes_per_second, one_second }; - CFArrayRef data_rate_limits = - CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); - OSStatus status = - VTSessionSetProperty(compression_session_, - kVTCompressionPropertyKey_DataRateLimits, - data_rate_limits); - if (bytes_per_second) { - CFRelease(bytes_per_second); - } - if (one_second) { - CFRelease(one_second); - } - if (data_rate_limits) { - CFRelease(data_rate_limits); - } - if (status != noErr) { - LOG(LS_ERROR) << "Failed to set data rate limit"; - } - - encoder_bitrate_bps_ = bitrate_bps; - } -} - -void H264VideoToolboxEncoder::OnEncodedFrame( - OSStatus status, - VTEncodeInfoFlags info_flags, - CMSampleBufferRef sample_buffer, - CodecSpecificInfo codec_specific_info, - int32_t width, - int32_t height, - int64_t render_time_ms, - uint32_t timestamp, - VideoRotation rotation) { - if (status != noErr) { - LOG(LS_ERROR) << "H264 encode failed."; - return; - } - if (info_flags & kVTEncodeInfo_FrameDropped) { - LOG(LS_INFO) << "H264 encode dropped frame."; - return; - } - - bool is_keyframe = false; - CFArrayRef attachments = - CMSampleBufferGetSampleAttachmentsArray(sample_buffer, 0); - if (attachments != nullptr && CFArrayGetCount(attachments)) { - CFDictionaryRef attachment = - static_cast(CFArrayGetValueAtIndex(attachments, 0)); - is_keyframe = - !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); - } - - if (is_keyframe) { - LOG(LS_INFO) << "Generated keyframe"; - } - - // Convert the sample buffer into a buffer suitable for RTP packetization. - // TODO(tkchin): Allocate buffers through a pool. - std::unique_ptr buffer(new rtc::Buffer()); - std::unique_ptr header; - { - webrtc::RTPFragmentationHeader* header_raw; - bool result = H264CMSampleBufferToAnnexBBuffer(sample_buffer, is_keyframe, - buffer.get(), &header_raw); - header.reset(header_raw); - if (!result) { - return; - } - } - webrtc::EncodedImage frame(buffer->data(), buffer->size(), buffer->size()); - frame._encodedWidth = width; - frame._encodedHeight = height; - frame._completeFrame = true; - frame._frameType = - is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta; - frame.capture_time_ms_ = render_time_ms; - frame._timeStamp = timestamp; - frame.rotation_ = rotation; - frame.content_type_ = - (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; - frame.timing_.is_timing_frame = false; - - h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); - h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); - - EncodedImageCallback::Result res = - callback_->OnEncodedImage(frame, &codec_specific_info, header.get()); - if (res.error != EncodedImageCallback::Result::OK) { - LOG(LS_ERROR) << "Encode callback failed: " << res.error; - return; - } - bitrate_adjuster_.Update(frame._length); -} - -// TODO(magjed): This function is not used by RTCVideoEncoderH264, but this whole file will be -// removed soon and inlined as ObjC. -VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() - const { - return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, - internal::kHighH264QpThreshold); -} -} // namespace webrtc diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/helpers.cc b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/helpers.cc new file mode 100644 index 0000000000..bd0619028b --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/helpers.cc @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "helpers.h" + +#include + +#include "webrtc/rtc_base/checks.h" +#include "webrtc/rtc_base/logging.h" + +// Copies characters from a CFStringRef into a std::string. +std::string CFStringToString(const CFStringRef cf_string) { + RTC_DCHECK(cf_string); + std::string std_string; + // Get the size needed for UTF8 plus terminating character. + size_t buffer_size = + CFStringGetMaximumSizeForEncoding(CFStringGetLength(cf_string), + kCFStringEncodingUTF8) + + 1; + std::unique_ptr buffer(new char[buffer_size]); + if (CFStringGetCString(cf_string, buffer.get(), buffer_size, + kCFStringEncodingUTF8)) { + // Copy over the characters. + std_string.assign(buffer.get()); + } + return std_string; +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + int32_t value) { + CFNumberRef cfNum = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &value); + OSStatus status = VTSessionSetProperty(session, key, cfNum); + CFRelease(cfNum); + if (status != noErr) { + std::string key_string = CFStringToString(key); + LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << value << ": " << status; + } +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + uint32_t value) { + int64_t value_64 = value; + CFNumberRef cfNum = + CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &value_64); + OSStatus status = VTSessionSetProperty(session, key, cfNum); + CFRelease(cfNum); + if (status != noErr) { + std::string key_string = CFStringToString(key); + LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << value << ": " << status; + } +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value) { + CFBooleanRef cf_bool = (value) ? kCFBooleanTrue : kCFBooleanFalse; + OSStatus status = VTSessionSetProperty(session, key, cf_bool); + if (status != noErr) { + std::string key_string = CFStringToString(key); + LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << value << ": " << status; + } +} + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + CFStringRef value) { + OSStatus status = VTSessionSetProperty(session, key, value); + if (status != noErr) { + std::string key_string = CFStringToString(key); + std::string val_string = CFStringToString(value); + LOG(LS_ERROR) << "VTSessionSetProperty failed to set: " << key_string + << " to " << val_string << ": " << status; + } +} diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/helpers.h b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/helpers.h new file mode 100644 index 0000000000..11c19ad7b4 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/helpers.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_HELPERS_H_ +#define WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_HELPERS_H_ + +#include +#include +#include + +// Convenience function for creating a dictionary. +inline CFDictionaryRef CreateCFTypeDictionary(CFTypeRef* keys, + CFTypeRef* values, + size_t size) { + return CFDictionaryCreate(kCFAllocatorDefault, keys, values, size, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); +} + +// Copies characters from a CFStringRef into a std::string. +std::string CFStringToString(const CFStringRef cf_string); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, CFStringRef key, int32_t value); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + uint32_t value); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, CFStringRef key, bool value); + +// Convenience function for setting a VT property. +void SetVTSessionProperty(VTSessionRef session, + CFStringRef key, + CFStringRef value); + +#endif // WEBRTC_SDK_OBJC_FRAMEWORK_CLASSES_VIDEOTOOLBOX_HELPERS_H_ diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm index f6e6ff55ac..5a1a51f3f5 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_decoder_factory.mm @@ -33,7 +33,6 @@ class ObjCVideoDecoder : public VideoDecoder { public: ObjCVideoDecoder(id decoder) : decoder_(decoder), implementation_name_([decoder implementationName].stdString) {} - ~ObjCVideoDecoder() { [decoder_ destroy]; } int32_t InitDecode(const VideoCodec *codec_settings, int32_t number_of_cores) { RTCVideoEncoderSettings *settings = diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm index 9c507fc9d1..c3de9ae706 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm @@ -54,7 +54,6 @@ class ObjCVideoEncoder : public VideoEncoder { public: ObjCVideoEncoder(id encoder) : encoder_(encoder), implementation_name_([encoder implementationName].stdString) {} - ~ObjCVideoEncoder() { [encoder_ destroy]; } int32_t InitEncode(const VideoCodec *codec_settings, int32_t number_of_cores, @@ -65,9 +64,9 @@ class ObjCVideoEncoder : public VideoEncoder { } int32_t RegisterEncodeCompleteCallback(EncodedImageCallback *callback) { - [encoder_ setCallback:^(RTCEncodedImage *frame, - id info, - RTCRtpFragmentationHeader *header) { + [encoder_ setCallback:^BOOL(RTCEncodedImage *_Nonnull frame, + id _Nonnull info, + RTCRtpFragmentationHeader *_Nonnull header) { EncodedImage encodedImage = [frame nativeEncodedImage]; // Handle types than can be converted into one of CodecSpecificInfo's hard coded cases. @@ -78,7 +77,9 @@ class ObjCVideoEncoder : public VideoEncoder { std::unique_ptr fragmentationHeader = [header createNativeFragmentationHeader]; - callback->OnEncodedImage(encodedImage, &codecSpecificInfo, fragmentationHeader.release()); + EncodedImageCallback::Result res = + callback->OnEncodedImage(encodedImage, &codecSpecificInfo, fragmentationHeader.release()); + return res.error == EncodedImageCallback::Result::OK; }]; return WEBRTC_VIDEO_CODEC_OK; @@ -113,11 +114,7 @@ class ObjCVideoEncoder : public VideoEncoder { int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } int32_t SetRates(uint32_t bitrate, uint32_t framerate) { - if ([encoder_ setBitrate:bitrate framerate:framerate]) { - return WEBRTC_VIDEO_CODEC_OK; - } else { - return WEBRTC_VIDEO_CODEC_ERROR; - } + return [encoder_ setBitrate:bitrate framerate:framerate]; } bool SupportsNativeHandle() const { return true; } diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h index a5214ea6c8..7f678b9759 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h @@ -18,9 +18,16 @@ NS_ASSUME_NONNULL_BEGIN /** Represents an encoded frame's type. */ typedef NS_ENUM(NSUInteger, RTCFrameType) { - RTCFrameTypeEmptyFrame, - RTCFrameTypeVideoFrameKey, - RTCFrameTypeVideoFrameDelta, + RTCFrameTypeEmptyFrame = 0, + RTCFrameTypeAudioFrameSpeech = 1, + RTCFrameTypeAudioFrameCN = 2, + RTCFrameTypeVideoFrameKey = 3, + RTCFrameTypeVideoFrameDelta = 4, +}; + +typedef NS_ENUM(NSUInteger, RTCVideoContentType) { + RTCVideoContentTypeUnspecified, + RTCVideoContentTypeScreenshare, }; /** Represents an encoded frame. Corresponds to webrtc::EncodedImage. */ @@ -40,6 +47,7 @@ RTC_EXPORT @property(nonatomic, assign) int rotation; @property(nonatomic, assign) BOOL completeFrame; @property(nonatomic, strong) NSNumber *qp; +@property(nonatomic, assign) RTCVideoContentType contentType; @end @@ -62,14 +70,32 @@ RTC_EXPORT @end +/** Class for H264 specific config. */ +typedef NS_ENUM(NSUInteger, RTCH264PacketizationMode) { + RTCH264PacketizationModeNonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed + RTCH264PacketizationModeSingleNalUnit // Mode 0 - only single NALU allowed +}; + +RTC_EXPORT +@interface RTCCodecSpecificInfoH264 : NSObject + +@property(nonatomic, assign) RTCH264PacketizationMode packetizationMode; + +@end + /** Callback block for encoder. */ -typedef void (^RTCVideoEncoderCallback)(RTCEncodedImage *frame, +typedef BOOL (^RTCVideoEncoderCallback)(RTCEncodedImage *frame, id info, RTCRtpFragmentationHeader *header); /** Callback block for decoder. */ typedef void (^RTCVideoDecoderCallback)(RTCVideoFrame *frame); +typedef NS_ENUM(NSUInteger, RTCVideoCodecMode) { + RTCVideoCodecModeRealtimeVideo, + RTCVideoCodecModeScreensharing, +}; + /** Holds information to identify a codec. Corresponds to cricket::VideoCodec. */ RTC_EXPORT @interface RTCVideoCodecInfo : NSObject @@ -101,6 +127,7 @@ RTC_EXPORT @property(nonatomic, assign) uint32_t maxFramerate; @property(nonatomic, assign) unsigned int qpMax; +@property(nonatomic, assign) RTCVideoCodecMode mode; @end @@ -123,11 +150,10 @@ RTC_EXPORT - (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings *)settings numberOfCores:(int)numberOfCores; - (NSInteger)releaseEncoder; -- (void)destroy; - (NSInteger)encode:(RTCVideoFrame *)frame codecSpecificInfo:(id)info frameTypes:(NSArray *)frameTypes; -- (BOOL)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate; +- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate; - (NSString *)implementationName; /** Returns QP scaling settings for encoder. The quality scaler adjusts the resolution in order to @@ -145,7 +171,6 @@ RTC_EXPORT - (NSInteger)startDecodeWithSettings:(RTCVideoEncoderSettings *)settings numberOfCores:(int)numberOfCores; - (NSInteger)releaseDecoder; -- (void)destroy; - (NSInteger)decode:(RTCEncodedImage *)encodedImage missingFrames:(BOOL)missingFrames fragmentationHeader:(RTCRtpFragmentationHeader *)fragmentationHeader diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h index fbb2bd29b6..449fcbba1b 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodecH264.h @@ -13,22 +13,12 @@ #import #import -/** Class for H264 specific config. */ -typedef NS_ENUM(NSUInteger, RTCH264PacketizationMode) { - NonInterleaved = 0, // Mode 1 - STAP-A, FU-A is allowed - SingleNalUnit // Mode 0 - only single NALU allowed -}; - -RTC_EXPORT -@interface RTCCodecSpecificInfoH264 : NSObject - -@property(nonatomic, assign) RTCH264PacketizationMode packetizationMode; - -@end - /** Encoder. */ RTC_EXPORT @interface RTCVideoEncoderH264 : NSObject + +- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo; + @end /** Decoder. */ diff --git a/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm b/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm index 11fa3fce9d..d5941e7abe 100644 --- a/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm +++ b/webrtc/sdk/objc/Framework/UnitTests/objc_video_encoder_factory_tests.mm @@ -29,7 +29,7 @@ id CreateEncoderFactoryReturning(int return_code) { OCMStub([encoderMock encode:[OCMArg any] codecSpecificInfo:[OCMArg any] frameTypes:[OCMArg any]]) .andReturn(return_code); OCMStub([encoderMock releaseEncoder]).andReturn(return_code); - OCMStub([encoderMock setBitrate:0 framerate:0]).andReturn(return_code == WEBRTC_VIDEO_CODEC_OK); + OCMStub([encoderMock setBitrate:0 framerate:0]).andReturn(return_code); id encoderFactoryMock = OCMProtocolMock(@protocol(RTCVideoEncoderFactory)); RTCVideoCodecInfo *supported =