From 5dfac33dfdd430d60f3ecf4c59d15a62ec1f68ef Mon Sep 17 00:00:00 2001 From: magjed Date: Tue, 1 Aug 2017 08:07:59 -0700 Subject: [PATCH] ObjC: Fix quality scaling for injected encoders We missed to implement quality scaling in the original CL https://codereview.webrtc.org/2977213002/. This CL implements it. Note that the ObjC interface for scalingSettings is slightly different from the C++ interface in that we require explicit QP thresholds to turn quality scaling on, i.e. we don't provide default values. I think this is more modular as we want to move codec specific knowledge out from the WebRTC core. I would like to update the C++ webrtc::VideoEncoder interface to do the same in another CL. BUG=webrtc:7924 Review-Url: https://codereview.webrtc.org/2991123002 Cr-Commit-Position: refs/heads/master@{#19202} --- .../Classes/PeerConnection/RTCVideoCodec.mm | 15 ++++++++++++++ .../PeerConnection/RTCVideoCodecH264.mm | 10 ++++++++++ .../Framework/Classes/VideoToolbox/encoder.mm | 2 ++ .../objc_video_encoder_factory.mm | 7 +++++++ .../Framework/Headers/WebRTC/RTCVideoCodec.h | 20 +++++++++++++++++++ 5 files changed, 54 insertions(+) diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm index 73c9a71480..1be33f0a8f 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodec.mm @@ -54,3 +54,18 @@ } @end + +@implementation RTCVideoEncoderQpThresholds + +@synthesize low = _low; +@synthesize high = _high; + +- (instancetype)initWithThresholdsLow:(NSInteger)low high:(NSInteger)high { + if (self = [super init]) { + _low = low; + _high = high; + } + return self; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm index 7cd6a7f0d0..7b38ec047c 100644 --- a/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm +++ b/webrtc/sdk/objc/Framework/Classes/PeerConnection/RTCVideoCodecH264.mm @@ -27,6 +27,11 @@ const size_t kDefaultPayloadSize = 1440; const char kHighProfileExperiment[] = "WebRTC-H264HighProfile"; +// These thresholds deviate from the default h264 QP thresholds, as they have been found to work +// better on devices that support VideoToolbox +const int kLowH264QpThreshold = 28; +const int kHighH264QpThreshold = 39; + bool IsHighProfileEnabled() { return webrtc::field_trial::IsEnabled(kHighProfileExperiment); } @@ -171,6 +176,11 @@ class H264VideoToolboxDecodeCompleteCallback : public webrtc::DecodedImageCallba return _videoToolboxEncoder->SetRates(bitrateKbit, framerate) == WEBRTC_VIDEO_CODEC_OK; } +- (RTCVideoEncoderQpThresholds *)scalingSettings { + return [[RTCVideoEncoderQpThresholds alloc] initWithThresholdsLow:kLowH264QpThreshold + high:kHighH264QpThreshold]; +} + @end // Decoder. diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm index d3f5ef9a93..2e87458fe8 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm @@ -757,6 +757,8 @@ void H264VideoToolboxEncoder::OnEncodedFrame( bitrate_adjuster_.Update(frame._length); } +// TODO(magjed): This function is not used by RTCVideoEncoderH264, but this whole file will be +// removed soon and inlined as ObjC. VideoEncoder::ScalingSettings H264VideoToolboxEncoder::GetScalingSettings() const { return VideoEncoder::ScalingSettings(true, internal::kLowH264QpThreshold, diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm index c464dabd79..0472fc07ae 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/objc_video_encoder_factory.mm @@ -111,6 +111,13 @@ class ObjCVideoEncoder : public VideoEncoder { bool SupportsNativeHandle() const { return true; } + VideoEncoder::ScalingSettings GetScalingSettings() const { + RTCVideoEncoderQpThresholds* qp_thresholds = [encoder_ scalingSettings]; + return qp_thresholds ? + ScalingSettings(true /* enabled */, qp_thresholds.low, qp_thresholds.high) : + ScalingSettings(false /* enabled */); + } + private: id encoder_; }; diff --git a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h index 95b519fdc0..a57e3e1077 100644 --- a/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h +++ b/webrtc/sdk/objc/Framework/Headers/WebRTC/RTCVideoCodec.h @@ -104,6 +104,17 @@ RTC_EXPORT @end +/** QP thresholds for encoder. Corresponds to webrtc::VideoEncoder::QpThresholds. */ +RTC_EXPORT +@interface RTCVideoEncoderQpThresholds : NSObject + +- (instancetype)initWithThresholdsLow:(NSInteger)low high:(NSInteger)high; + +@property(nonatomic, readonly) NSInteger low; +@property(nonatomic, readonly) NSInteger high; + +@end + /** Protocol for encoder implementations. */ RTC_EXPORT @protocol RTCVideoEncoder @@ -118,6 +129,13 @@ RTC_EXPORT frameTypes:(NSArray *)frameTypes; - (BOOL)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate; +/** Returns QP scaling settings for encoder. The quality scaler adjusts the resolution in order to + * keep the QP from the encoded images within the given range. Returning nil from this function + * disables quality scaling. */ +- (RTCVideoEncoderQpThresholds *)scalingSettings; + +// TODO(andersc): Add implementationName method. + @end /** Protocol for decoder implementations. */ @@ -135,6 +153,8 @@ RTC_EXPORT codecSpecificInfo:(__nullable id)info renderTimeMs:(int64_t)renderTimeMs; +// TODO(andersc): Add implementationName method. + @end NS_ASSUME_NONNULL_END