From da2c4cede08a3758ea97c50221d124a0866cdb3c Mon Sep 17 00:00:00 2001 From: "pbos@webrtc.org" Date: Tue, 17 Sep 2013 09:38:41 +0000 Subject: [PATCH] Fix dangling pointer _encoder in video_sender.cc. When _codecDataBase.SetSendCodec() fails, the encoder may be deleted. This is however not reflected in _encoder, which then becomes a dangling pointer to the deleted object. BUG=2384 R=stefan@webrtc.org Review URL: https://webrtc-codereview.appspot.com/2229004 git-svn-id: http://webrtc.googlecode.com/svn/trunk@4765 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../video_coding/main/source/video_coding_impl.h | 1 - .../modules/video_coding/main/source/video_sender.cc | 12 +++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/webrtc/modules/video_coding/main/source/video_coding_impl.h b/webrtc/modules/video_coding/main/source/video_coding_impl.h index 21f5c32d91..1ead48d628 100644 --- a/webrtc/modules/video_coding/main/source/video_coding_impl.h +++ b/webrtc/modules/video_coding/main/source/video_coding_impl.h @@ -107,7 +107,6 @@ class VideoSender { VCMEncodedFrameCallback _encodedFrameCallback; std::vector _nextFrameTypes; media_optimization::VCMMediaOptimization _mediaOpt; - VideoCodecType _sendCodecType; VCMSendStatisticsCallback* _sendStatsCallback; FILE* _encoderInputFile; VCMCodecDataBase _codecDataBase; diff --git a/webrtc/modules/video_coding/main/source/video_sender.cc b/webrtc/modules/video_coding/main/source/video_sender.cc index 73ee856230..eb06133526 100644 --- a/webrtc/modules/video_coding/main/source/video_sender.cc +++ b/webrtc/modules/video_coding/main/source/video_sender.cc @@ -27,7 +27,6 @@ VideoSender::VideoSender(const int32_t id, Clock* clock) _encodedFrameCallback(), _nextFrameTypes(1, kVideoFrameDelta), _mediaOpt(id, clock_), - _sendCodecType(kVideoCodecUnknown), _sendStatsCallback(NULL), _encoderInputFile(NULL), _codecDataBase(id), @@ -88,6 +87,11 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, bool ret = _codecDataBase.SetSendCodec( sendCodec, numberOfCores, maxPayloadSize, &_encodedFrameCallback); + + // Update encoder regardless of result to make sure that we're not holding on + // to a deleted instance. + _encoder = _codecDataBase.GetEncoder(); + if (!ret) { WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideoCoding, @@ -96,9 +100,7 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, return VCM_CODEC_ERROR; } - _encoder = _codecDataBase.GetEncoder(); - _sendCodecType = sendCodec->codecType; - int numLayers = (_sendCodecType != kVideoCodecVP8) + int numLayers = (sendCodec->codecType != kVideoCodecVP8) ? 1 : sendCodec->codecSpecific.VP8.numberOfTemporalLayers; // If we have screensharing and we have layers, we disable frame dropper. @@ -113,7 +115,7 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, _nextFrameTypes.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), kVideoFrameDelta); - _mediaOpt.SetEncodingData(_sendCodecType, + _mediaOpt.SetEncodingData(sendCodec->CodecType, sendCodec->maxBitrate * 1000, sendCodec->maxFramerate * 1000, sendCodec->startBitrate * 1000,