From 5ed5ed953d2d0d667892a0bf528f65eb430cab18 Mon Sep 17 00:00:00 2001 From: tkchin Date: Tue, 8 Mar 2016 10:51:54 -0800 Subject: [PATCH] Fix VideoToolbox backgrounding issues. When the iOS application is not in the foreground, the hardware encoder and decoder become invalidated. There doesn't seem to be a way to query their state so we don't know they're invalid until we get an error code after an encode/decode request. To solve the issue, we just don't encode/decode when the app is not active, and reinitialize the encoder/decoder when the app is active again. Also fixes a leak in the decoder. BUG=webrtc:4081 Review URL: https://codereview.webrtc.org/1732953003 Cr-Commit-Position: refs/heads/master@{#11916} --- webrtc/base/BUILD.gn | 2 + webrtc/base/base.gyp | 2 + webrtc/base/objc/RTCUIApplication.h | 21 +++++ webrtc/base/objc/RTCUIApplication.mm | 22 +++++ webrtc/build/WebRTC-Prefix.pch | 6 +- .../video_coding/codecs/h264/h264.gypi | 4 + .../codecs/h264/h264_video_toolbox_decoder.cc | 57 ++++++++++-- .../codecs/h264/h264_video_toolbox_encoder.cc | 32 ++++++- .../codecs/h264/h264_video_toolbox_nalu.cc | 92 +++++++++++-------- .../codecs/h264/h264_video_toolbox_nalu.h | 12 +++ .../h264/h264_video_toolbox_nalu_unittest.cc | 35 +++++++ 11 files changed, 237 insertions(+), 48 deletions(-) create mode 100644 webrtc/base/objc/RTCUIApplication.h create mode 100644 webrtc/base/objc/RTCUIApplication.mm diff --git a/webrtc/base/BUILD.gn b/webrtc/base/BUILD.gn index ac2a3ac59c..b4d14fc6d1 100644 --- a/webrtc/base/BUILD.gn +++ b/webrtc/base/BUILD.gn @@ -649,6 +649,8 @@ if (is_ios) { "objc/RTCSSLAdapter.mm", "objc/RTCTracing.h", "objc/RTCTracing.mm", + "objc/RTCUIApplication.h", + "objc/RTCUIApplication.mm", ] } } diff --git a/webrtc/base/base.gyp b/webrtc/base/base.gyp index eef8c2acba..f43fde215c 100644 --- a/webrtc/base/base.gyp +++ b/webrtc/base/base.gyp @@ -53,6 +53,8 @@ 'sources': [ 'objc/RTCCameraPreviewView.h', 'objc/RTCCameraPreviewView.m', + 'objc/RTCUIApplication.h', + 'objc/RTCUIApplication.mm', ], 'all_dependent_settings': { 'xcode_settings': { diff --git a/webrtc/base/objc/RTCUIApplication.h b/webrtc/base/objc/RTCUIApplication.h new file mode 100644 index 0000000000..c06404f777 --- /dev/null +++ b/webrtc/base/objc/RTCUIApplication.h @@ -0,0 +1,21 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_BASE_OBJC_RTC_UI_APPLICATION_H_ +#define WEBRTC_BASE_OBJC_RTC_UI_APPLICATION_H_ + +#include "webrtc/base/objc/RTCMacros.h" + +#if defined(WEBRTC_IOS) +/** Convenience function to get UIApplicationState from C++. */ +RTC_EXPORT bool RTCIsUIApplicationActive(); +#endif // WEBRTC_IOS + +#endif // WEBRTC_BASE_OBJC_RTC_UI_APPLICATION_H_ diff --git a/webrtc/base/objc/RTCUIApplication.mm b/webrtc/base/objc/RTCUIApplication.mm new file mode 100644 index 0000000000..85376b4e0e --- /dev/null +++ b/webrtc/base/objc/RTCUIApplication.mm @@ -0,0 +1,22 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/base/objc/RTCUIApplication.h" + +#if defined(WEBRTC_IOS) + +#import + +bool RTCIsUIApplicationActive() { + UIApplicationState state = [UIApplication sharedApplication].applicationState; + return state == UIApplicationStateActive; +} + +#endif // WEBRTC_IOS diff --git a/webrtc/build/WebRTC-Prefix.pch b/webrtc/build/WebRTC-Prefix.pch index ab1bc7a20d..506d4fbb75 100644 --- a/webrtc/build/WebRTC-Prefix.pch +++ b/webrtc/build/WebRTC-Prefix.pch @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#ifdef __OBJC__ + #if !defined(__has_feature) || !__has_feature(objc_arc) #error "This file requires ARC support." #endif @@ -19,8 +21,6 @@ // The following nonnull macros were introduced in OSX SDK 10.10.3. However, // the bots appear to be running something older. We define them here if they // aren't already defined in NSObjCRuntime.h -#ifdef __OBJC__ - #include #if !defined(NS_ASSUME_NONNULL_BEGIN) @@ -31,4 +31,4 @@ #define NS_ASSUME_NONNULL_END #endif -#endif +#endif // __OBJC__ diff --git a/webrtc/modules/video_coding/codecs/h264/h264.gypi b/webrtc/modules/video_coding/codecs/h264/h264.gypi index 1d493daa98..9cb58a8598 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264.gypi +++ b/webrtc/modules/video_coding/codecs/h264/h264.gypi @@ -62,6 +62,10 @@ { 'target_name': 'webrtc_h264_video_toolbox', 'type': 'static_library', + 'includes': [ '../../../../build/objc_common.gypi' ], + 'dependencies': [ + '<(webrtc_root)/base/base.gyp:rtc_base_objc', + ], 'link_settings': { 'xcode_settings': { 'OTHER_LDFLAGS': [ diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc index eb11c59434..5db2cd1e9e 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc @@ -18,6 +18,9 @@ #include "libyuv/convert.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#if defined(WEBRTC_IOS) +#include "webrtc/base/objc/RTCUIApplication.h" +#endif #include "webrtc/common_video/include/video_frame_buffer.h" #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" #include "webrtc/video_frame.h" @@ -128,6 +131,40 @@ int H264VideoToolboxDecoder::Decode( int64_t render_time_ms) { RTC_DCHECK(input_image._buffer); +#if defined(WEBRTC_IOS) + if (!RTCIsUIApplicationActive()) { + // Ignore all decode requests when app isn't active. In this state, the + // hardware decoder has been invalidated by the OS. + // Reset video format so that we won't process frames until the next + // keyframe. + SetVideoFormat(nullptr); + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } +#endif + CMVideoFormatDescriptionRef input_format = nullptr; + if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer, + input_image._length)) { + input_format = CreateVideoFormatDescription(input_image._buffer, + input_image._length); + if (input_format) { + // Check if the video format has changed, and reinitialize decoder if + // needed. + if (!CMFormatDescriptionEqual(input_format, video_format_)) { + SetVideoFormat(input_format); + ResetDecompressionSession(); + } + CFRelease(input_format); + } + } + if (!video_format_) { + // We received a frame but we don't have format information so we can't + // decode it. + // This can happen after backgrounding. We need to wait for the next + // sps/pps before we can resume so we request a keyframe by returning an + // error. + LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; + return WEBRTC_VIDEO_CODEC_ERROR; + } CMSampleBufferRef sample_buffer = nullptr; if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer, input_image._length, video_format_, @@ -135,13 +172,6 @@ int H264VideoToolboxDecoder::Decode( return WEBRTC_VIDEO_CODEC_ERROR; } RTC_DCHECK(sample_buffer); - // Check if the video format has changed, and reinitialize decoder if needed. - CMVideoFormatDescriptionRef description = - CMSampleBufferGetFormatDescription(sample_buffer); - if (!CMFormatDescriptionEqual(description, video_format_)) { - SetVideoFormat(description); - ResetDecompressionSession(); - } VTDecodeFrameFlags decode_flags = kVTDecodeFrame_EnableAsynchronousDecompression; std::unique_ptr frame_decode_params; @@ -150,6 +180,18 @@ int H264VideoToolboxDecoder::Decode( OSStatus status = VTDecompressionSessionDecodeFrame( decompression_session_, sample_buffer, decode_flags, frame_decode_params.release(), nullptr); +#if defined(WEBRTC_IOS) + // Re-initialize the decoder if we have an invalid session while the app is + // active and retry the decode request. + if (status == kVTInvalidSessionErr && + ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) { + frame_decode_params.reset( + new internal::FrameDecodeParams(callback_, input_image._timeStamp)); + status = VTDecompressionSessionDecodeFrame( + decompression_session_, sample_buffer, decode_flags, + frame_decode_params.release(), nullptr); + } +#endif CFRelease(sample_buffer); if (status != noErr) { LOG(LS_ERROR) << "Failed to decode frame with code: " << status; @@ -244,6 +286,7 @@ void H264VideoToolboxDecoder::ConfigureDecompressionSession() { void H264VideoToolboxDecoder::DestroyDecompressionSession() { if (decompression_session_) { VTDecompressionSessionInvalidate(decompression_session_); + CFRelease(decompression_session_); decompression_session_ = nullptr; } } diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc index 407bb67608..8cfe63dbb7 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc @@ -20,6 +20,9 @@ #include "libyuv/convert_from.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#if defined(WEBRTC_IOS) +#include "webrtc/base/objc/RTCUIApplication.h" +#endif #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" #include "webrtc/system_wrappers/include/clock.h" @@ -238,10 +241,31 @@ int H264VideoToolboxEncoder::Encode( if (!callback_ || !compression_session_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - +#if defined(WEBRTC_IOS) + if (!RTCIsUIApplicationActive()) { + // Ignore all encode requests when app isn't active. In this state, the + // hardware encoder has been invalidated by the OS. + return WEBRTC_VIDEO_CODEC_OK; + } +#endif // Get a pixel buffer from the pool and copy frame data over. CVPixelBufferPoolRef pixel_buffer_pool = VTCompressionSessionGetPixelBufferPool(compression_session_); +#if defined(WEBRTC_IOS) + if (!pixel_buffer_pool) { + // Kind of a hack. On backgrounding, the compression session seems to get + // invalidated, which causes this pool call to fail when the application + // is foregrounded and frames are being sent for encoding again. + // Resetting the session when this happens fixes the issue. + ResetCompressionSession(); + pixel_buffer_pool = + VTCompressionSessionGetPixelBufferPool(compression_session_); + } +#endif + if (!pixel_buffer_pool) { + LOG(LS_ERROR) << "Failed to get pixel buffer pool."; + return WEBRTC_VIDEO_CODEC_ERROR; + } CVPixelBufferRef pixel_buffer = nullptr; CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, &pixel_buffer); @@ -285,7 +309,7 @@ int H264VideoToolboxEncoder::Encode( // Update the bitrate if needed. SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps()); - VTCompressionSessionEncodeFrame( + OSStatus status = VTCompressionSessionEncodeFrame( compression_session_, pixel_buffer, presentation_time_stamp, kCMTimeInvalid, frame_properties, encode_params.release(), nullptr); if (frame_properties) { @@ -294,6 +318,10 @@ int H264VideoToolboxEncoder::Encode( if (pixel_buffer) { CVBufferRelease(pixel_buffer); } + if (status != noErr) { + LOG(LS_ERROR) << "Failed to encode frame with code: " << status; + return WEBRTC_VIDEO_CODEC_ERROR; + } return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.cc index 8328a56cb7..b0c4bd326d 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.cc @@ -161,55 +161,34 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, CMSampleBufferRef* out_sample_buffer) { RTC_DCHECK(annexb_buffer); RTC_DCHECK(out_sample_buffer); + RTC_DCHECK(video_format); *out_sample_buffer = nullptr; - // The buffer we receive via RTP has 00 00 00 01 start code artifically - // embedded by the RTP depacketizer. Extract NALU information. - // TODO(tkchin): handle potential case where sps and pps are delivered - // separately. - uint8_t first_nalu_type = annexb_buffer[4] & 0x1f; - bool is_first_nalu_type_sps = first_nalu_type == 0x7; - AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size); - CMVideoFormatDescriptionRef description = nullptr; - OSStatus status = noErr; - if (is_first_nalu_type_sps) { - // Parse the SPS and PPS into a CMVideoFormatDescription. - const uint8_t* param_set_ptrs[2] = {}; - size_t param_set_sizes[2] = {}; - if (!reader.ReadNalu(¶m_set_ptrs[0], ¶m_set_sizes[0])) { + if (H264AnnexBBufferHasVideoFormatDescription(annexb_buffer, + annexb_buffer_size)) { + // Advance past the SPS and PPS. + const uint8_t* data = nullptr; + size_t data_len = 0; + if (!reader.ReadNalu(&data, &data_len)) { LOG(LS_ERROR) << "Failed to read SPS"; return false; } - if (!reader.ReadNalu(¶m_set_ptrs[1], ¶m_set_sizes[1])) { + if (!reader.ReadNalu(&data, &data_len)) { LOG(LS_ERROR) << "Failed to read PPS"; return false; } - status = CMVideoFormatDescriptionCreateFromH264ParameterSets( - kCFAllocatorDefault, 2, param_set_ptrs, param_set_sizes, 4, - &description); - if (status != noErr) { - LOG(LS_ERROR) << "Failed to create video format description."; - return false; - } - } else { - RTC_DCHECK(video_format); - description = video_format; - // We don't need to retain, but it makes logic easier since we are creating - // in the other block. - CFRetain(description); } // Allocate memory as a block buffer. // TODO(tkchin): figure out how to use a pool. CMBlockBufferRef block_buffer = nullptr; - status = CMBlockBufferCreateWithMemoryBlock( + OSStatus status = CMBlockBufferCreateWithMemoryBlock( nullptr, nullptr, reader.BytesRemaining(), nullptr, nullptr, 0, reader.BytesRemaining(), kCMBlockBufferAssureMemoryNowFlag, &block_buffer); if (status != kCMBlockBufferNoErr) { LOG(LS_ERROR) << "Failed to create block buffer."; - CFRelease(description); return false; } @@ -221,7 +200,6 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, if (status != noErr) { LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: " << status; - CFRelease(description); CFRelease(block_buffer); return false; } @@ -237,7 +215,6 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, &block_buffer_size, &data_ptr); if (status != kCMBlockBufferNoErr) { LOG(LS_ERROR) << "Failed to get block buffer data pointer."; - CFRelease(description); CFRelease(contiguous_buffer); return false; } @@ -256,19 +233,62 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, // Create sample buffer. status = CMSampleBufferCreate(nullptr, contiguous_buffer, true, nullptr, - nullptr, description, 1, 0, nullptr, 0, nullptr, - out_sample_buffer); + nullptr, video_format, 1, 0, nullptr, 0, + nullptr, out_sample_buffer); if (status != noErr) { LOG(LS_ERROR) << "Failed to create sample buffer."; - CFRelease(description); CFRelease(contiguous_buffer); return false; } - CFRelease(description); CFRelease(contiguous_buffer); return true; } +bool H264AnnexBBufferHasVideoFormatDescription(const uint8_t* annexb_buffer, + size_t annexb_buffer_size) { + RTC_DCHECK(annexb_buffer); + RTC_DCHECK_GT(annexb_buffer_size, 4u); + + // The buffer we receive via RTP has 00 00 00 01 start code artifically + // embedded by the RTP depacketizer. Extract NALU information. + // TODO(tkchin): handle potential case where sps and pps are delivered + // separately. + uint8_t first_nalu_type = annexb_buffer[4] & 0x1f; + bool is_first_nalu_type_sps = first_nalu_type == 0x7; + return is_first_nalu_type_sps; +} + +CMVideoFormatDescriptionRef CreateVideoFormatDescription( + const uint8_t* annexb_buffer, + size_t annexb_buffer_size) { + if (!H264AnnexBBufferHasVideoFormatDescription(annexb_buffer, + annexb_buffer_size)) { + return nullptr; + } + AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size); + CMVideoFormatDescriptionRef description = nullptr; + OSStatus status = noErr; + // Parse the SPS and PPS into a CMVideoFormatDescription. + const uint8_t* param_set_ptrs[2] = {}; + size_t param_set_sizes[2] = {}; + if (!reader.ReadNalu(¶m_set_ptrs[0], ¶m_set_sizes[0])) { + LOG(LS_ERROR) << "Failed to read SPS"; + return nullptr; + } + if (!reader.ReadNalu(¶m_set_ptrs[1], ¶m_set_sizes[1])) { + LOG(LS_ERROR) << "Failed to read PPS"; + return nullptr; + } + status = CMVideoFormatDescriptionCreateFromH264ParameterSets( + kCFAllocatorDefault, 2, param_set_ptrs, param_set_sizes, 4, + &description); + if (status != noErr) { + LOG(LS_ERROR) << "Failed to create video format description."; + return nullptr; + } + return description; +} + AnnexBBufferReader::AnnexBBufferReader(const uint8_t* annexb_buffer, size_t length) : start_(annexb_buffer), offset_(0), next_offset_(0), length_(length) { diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h index 31ef525816..50942536f3 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h @@ -44,6 +44,18 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer, CMVideoFormatDescriptionRef video_format, CMSampleBufferRef* out_sample_buffer); +// Returns true if the type of the first NALU in the supplied Annex B buffer is +// the SPS type. +bool H264AnnexBBufferHasVideoFormatDescription(const uint8_t* annexb_buffer, + size_t annexb_buffer_size); + +// Returns a video format description created from the sps/pps information in +// the Annex B buffer. If there is no such information, nullptr is returned. +// The caller is responsible for releasing the description. +CMVideoFormatDescriptionRef CreateVideoFormatDescription( + const uint8_t* annexb_buffer, + size_t annexb_buffer_size); + // Helper class for reading NALUs from an RTP Annex B buffer. class AnnexBBufferReader final { public: diff --git a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu_unittest.cc b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu_unittest.cc index 4087dbc79d..4ac9cd0db1 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu_unittest.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu_unittest.cc @@ -16,11 +16,44 @@ #include "webrtc/base/arraysize.h" #include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h" +#if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED) + namespace webrtc { static const uint8_t NALU_TEST_DATA_0[] = {0xAA, 0xBB, 0xCC}; static const uint8_t NALU_TEST_DATA_1[] = {0xDE, 0xAD, 0xBE, 0xEF}; +TEST(H264VideoToolboxNaluTest, TestHasVideoFormatDescription) { + const uint8_t sps_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x27}; + EXPECT_TRUE(H264AnnexBBufferHasVideoFormatDescription(sps_buffer, + arraysize(sps_buffer))); + const uint8_t other_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x28}; + EXPECT_FALSE(H264AnnexBBufferHasVideoFormatDescription( + other_buffer, arraysize(other_buffer))); +} + +TEST(H264VideoToolboxNaluTest, TestCreateVideoFormatDescription) { + const uint8_t sps_pps_buffer[] = { + // SPS nalu. + 0x00, 0x00, 0x00, 0x01, + 0x27, 0x42, 0x00, 0x1E, 0xAB, 0x40, 0xF0, 0x28, 0xD3, 0x70, 0x20, 0x20, + 0x20, 0x20, + // PPS nalu. + 0x00, 0x00, 0x00, 0x01, + 0x28, 0xCE, 0x3C, 0x30 + }; + CMVideoFormatDescriptionRef description = + CreateVideoFormatDescription(sps_pps_buffer, arraysize(sps_pps_buffer)); + EXPECT_TRUE(description); + if (description) { + CFRelease(description); + description = nullptr; + } + const uint8_t other_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x28}; + EXPECT_FALSE(CreateVideoFormatDescription(other_buffer, + arraysize(other_buffer))); +} + TEST(AnnexBBufferReaderTest, TestReadEmptyInput) { const uint8_t annex_b_test_data[] = {0x00}; AnnexBBufferReader reader(annex_b_test_data, 0); @@ -151,3 +184,5 @@ TEST(AvccBufferWriterTest, TestOverflow) { } } // namespace webrtc + +#endif // WEBRTC_VIDEO_TOOLBOX_SUPPORTED