From 604abe09f1d8bffc9e0d52c09484371f063cc2cd Mon Sep 17 00:00:00 2001 From: magjed Date: Thu, 19 May 2016 06:05:40 -0700 Subject: [PATCH] VideoAdapter: Drop frames based on actual fps instead of expected fps Pass timestamps to VideoAdapter instead of setting expected input frame rate, and use that to calculate when frames should be dropped. BUG=webrtc:4938 TEST=Enable quality slider and HUD in debug settings. Request low fps with the quality slider and observe dropped frames. Review-Url: https://codereview.webrtc.org/1982983003 Cr-Commit-Position: refs/heads/master@{#12811} --- webrtc/media/base/videoadapter.cc | 60 ++-- webrtc/media/base/videoadapter.h | 13 +- webrtc/media/base/videoadapter_unittest.cc | 311 +++++++++++++++------ webrtc/media/base/videocapturer.cc | 1 + webrtc/media/base/videocapturer.h | 5 - 5 files changed, 268 insertions(+), 122 deletions(-) diff --git a/webrtc/media/base/videoadapter.cc b/webrtc/media/base/videoadapter.cc index 9c3837c572..e1f506f10b 100644 --- a/webrtc/media/base/videoadapter.cc +++ b/webrtc/media/base/videoadapter.cc @@ -11,6 +11,7 @@ #include "webrtc/media/base/videoadapter.h" #include +#include #include #include "webrtc/base/checks.h" @@ -118,22 +119,43 @@ VideoAdapter::VideoAdapter() adaption_changes_(0), previous_width_(0), previous_height_(0), - input_interval_(0), - interval_next_frame_(0), resolution_request_max_pixel_count_(std::numeric_limits::max()), resolution_request_max_pixel_count_step_up_(0) {} VideoAdapter::~VideoAdapter() {} -void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) { - // TODO(perkj): Consider measuring input frame rate instead. - // Frame rate typically varies depending on lighting. +bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) { rtc::CritScope cs(&critical_section_); - input_interval_ = interval; + if (!requested_format_ || requested_format_->interval == 0) + return true; + + if (next_frame_timestamp_ns_) { + // Time until next frame should be outputted. + const int64_t time_until_next_frame_ns = + (*next_frame_timestamp_ns_ - in_timestamp_ns); + + // Continue if timestamp is withing expected range. + if (std::abs(time_until_next_frame_ns) < 2 * requested_format_->interval) { + // Drop if a frame shouldn't be outputted yet. + if (time_until_next_frame_ns > 0) + return false; + // Time to output new frame. + *next_frame_timestamp_ns_ += requested_format_->interval; + return true; + } + } + + // First timestamp received or timestamp is way outside expected range, so + // reset. Set first timestamp target to just half the interval to prefer + // keeping frames in case of jitter. + next_frame_timestamp_ns_ = + rtc::Optional(in_timestamp_ns + requested_format_->interval / 2); + return true; } void VideoAdapter::AdaptFrameResolution(int in_width, int in_height, + int64_t in_timestamp_ns, int* cropped_width, int* cropped_height, int* out_width, @@ -150,25 +172,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, } // Drop the input frame if necessary. - bool should_drop = false; - if (max_pixel_count == 0) { - // Drop all frames as the output format is 0x0. - should_drop = true; - } else if (requested_format_ && requested_format_->interval > 0) { - // Drop some frames based on input fps and output fps. - // Normally output fps is less than input fps. - interval_next_frame_ += input_interval_; - if (interval_next_frame_ >= requested_format_->interval) { - interval_next_frame_ -= requested_format_->interval; - // Reset |interval_next_frame_| if it accumulates too much to avoid - // "catching up" behaviour. - if (interval_next_frame_ >= requested_format_->interval) - interval_next_frame_ = 0; - } else { - should_drop = true; - } - } - if (should_drop) { + if (max_pixel_count == 0 || !KeepFrame(in_timestamp_ns)) { // Show VAdapt log every 90 frames dropped. (3 seconds) if ((frames_in_ - frames_out_) % 90 == 0) { // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed @@ -179,7 +183,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, << " Changes: " << adaption_changes_ << " Input: " << in_width << "x" << in_height - << " i" << input_interval_ + << " timestamp: " << in_timestamp_ns << " Output: i" << (requested_format_ ? requested_format_->interval : 0); } @@ -238,7 +242,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out " << frames_out_ << " / in " << frames_in_ << " Changes: " << adaption_changes_ << " Input: " << in_width - << "x" << in_height << " i" << input_interval_ + << "x" << in_height << " Scale: " << scale.numerator << "/" << scale.denominator << " Output: " << *out_width << "x" << *out_height << " i" << (requested_format_ ? requested_format_->interval : 0); @@ -251,7 +255,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { rtc::CritScope cs(&critical_section_); requested_format_ = rtc::Optional(format); - interval_next_frame_ = 0; + next_frame_timestamp_ns_ = rtc::Optional(); } void VideoAdapter::OnResolutionRequest( diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h index 2db0ada271..45fd1fa827 100644 --- a/webrtc/media/base/videoadapter.h +++ b/webrtc/media/base/videoadapter.h @@ -27,17 +27,13 @@ class VideoAdapter { VideoAdapter(); virtual ~VideoAdapter(); - // Sets the expected frame interval. This controls how often frames should - // be dropped if |OnOutputFormatRequest| is called with a lower frame - // interval. - void SetExpectedInputFrameInterval(int64_t interval); - // Return the adapted resolution given the input resolution. The input // resolution should first be cropped to the specified resolution, and then // scaled to the final output resolution. The output resolution will be 0x0 if // the frame should be dropped. void AdaptFrameResolution(int in_width, int in_height, + int64_t in_timestamp_ns, int* cropped_width, int* cropped_height, int* out_width, @@ -58,14 +54,17 @@ class VideoAdapter { rtc::Optional max_pixel_count_step_up); private: + // Determine if frame should be dropped based on input fps and requested fps. + bool KeepFrame(int64_t in_timestamp_ns); + int frames_in_; // Number of input frames. int frames_out_; // Number of output frames. int frames_scaled_; // Number of frames scaled. int adaption_changes_; // Number of changes in scale factor. int previous_width_; // Previous adapter output width. int previous_height_; // Previous adapter output height. - int input_interval_ GUARDED_BY(critical_section_); - int64_t interval_next_frame_ GUARDED_BY(critical_section_); + // The target timestamp for the next frame based on requested format. + rtc::Optional next_frame_timestamp_ns_ GUARDED_BY(critical_section_); // Max number of pixels requested via calls to OnOutputFormatRequest, // OnResolutionRequest respectively. diff --git a/webrtc/media/base/videoadapter_unittest.cc b/webrtc/media/base/videoadapter_unittest.cc index 92b4a558b9..d76bbd6ac8 100644 --- a/webrtc/media/base/videoadapter_unittest.cc +++ b/webrtc/media/base/videoadapter_unittest.cc @@ -28,8 +28,7 @@ class VideoAdapterTest : public testing::Test { virtual void SetUp() { capturer_.reset(new FakeVideoCapturer); capture_format_ = capturer_->GetSupportedFormats()->at(0); - capture_format_.interval = VideoFormat::FpsToInterval(50); - adapter_.SetExpectedInputFrameInterval(capture_format_.interval); + capture_format_.interval = VideoFormat::FpsToInterval(30); listener_.reset(new VideoCapturerListener(&adapter_)); capturer_->SignalFrameCaptured.connect( @@ -73,6 +72,7 @@ class VideoAdapterTest : public testing::Test { int out_width; int out_height; video_adapter_->AdaptFrameResolution(in_width, in_height, + captured_frame->time_stamp, &cropped_width, &cropped_height, &out_width, &out_height); if (out_width != 0 && out_height != 0) { @@ -170,39 +170,193 @@ TEST_F(VideoAdapterTest, AdaptZeroInterval) { // Adapt the frame rate to be half of the capture rate at the beginning. Expect // the number of dropped frames to be half of the number the captured frames. -TEST_F(VideoAdapterTest, AdaptFramerate) { +TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { VideoFormat request_format = capture_format_; request_format.interval *= 2; adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); + + // Capture 10 frames and verify that every other frame is dropped. The first + // frame should not be dropped. + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 1); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 2); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 3); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 4); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 5); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 6); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 7); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 8); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 9); + EXPECT_EQ(4, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 10); + EXPECT_EQ(4, listener_->GetStats().dropped_frames); +} + +// Adapt the frame rate to be two thirds of the capture rate at the beginning. +// Expect the number of dropped frames to be one thirds of the number the +// captured frames. +TEST_F(VideoAdapterTest, AdaptFramerateToTwoThirds) { + VideoFormat request_format = capture_format_; + request_format.interval = request_format.interval * 3 / 2; + adapter_.OnOutputFormatRequest(request_format); + EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); + + // Capture 10 frames and verify that every third frame is dropped. The first + // frame should not be dropped. + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 1); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 2); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 3); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 4); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 5); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 6); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 7); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 8); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 9); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 10); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); +} + +// Request frame rate twice as high as captured frame rate. Expect no frame +// drop. +TEST_F(VideoAdapterTest, AdaptFramerateHighLimit) { + VideoFormat request_format = capture_format_; + request_format.interval /= 2; + adapter_.OnOutputFormatRequest(request_format); + EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); - // Verify frame drop and no resolution change. - VideoCapturerListener::Stats stats = listener_->GetStats(); - EXPECT_GE(stats.captured_frames, 10); - EXPECT_EQ(stats.captured_frames / 2, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, - capture_format_.width, capture_format_.height); + // Verify no frame drop. + EXPECT_EQ(0, listener_->GetStats().dropped_frames); } -// Adapt the frame rate to be half of the capture rate at the beginning. Expect -// the number of dropped frames to be half of the number the captured frames. -TEST_F(VideoAdapterTest, AdaptFramerateVariable) { - VideoFormat request_format = capture_format_; - request_format.interval = request_format.interval * 3 / 2; - adapter_.OnOutputFormatRequest(request_format); - EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); - for (int i = 0; i < 30; ++i) - capturer_->CaptureFrame(); +// After the first timestamp, add a big offset to the timestamps. Expect that +// the adapter is conservative and resets to the new offset and does not drop +// any frame. +TEST_F(VideoAdapterTest, AdaptFramerateTimestampOffset) { + const int64_t capture_interval = VideoFormat::FpsToInterval(30); + adapter_.OnOutputFormatRequest( + VideoFormat(640, 480, capture_interval, cricket::FOURCC_ANY)); - // Verify frame drop and no resolution change. - VideoCapturerListener::Stats stats = listener_->GetStats(); - EXPECT_GE(stats.captured_frames, 30); - // Verify 2 / 3 kept (20) and 1 / 3 dropped (10). - EXPECT_EQ(stats.captured_frames * 1 / 3, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, - capture_format_.width, capture_format_.height); + const int64_t first_timestamp = 0; + adapter_.AdaptFrameResolution(640, 480, first_timestamp, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + const int64_t big_offset = -987654321LL * 1000; + const int64_t second_timestamp = big_offset; + adapter_.AdaptFrameResolution(640, 480, second_timestamp, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + const int64_t third_timestamp = big_offset + capture_interval; + adapter_.AdaptFrameResolution(640, 480, third_timestamp, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); +} + +// Request 30 fps and send 30 fps with jitter. Expect that no frame is dropped. +TEST_F(VideoAdapterTest, AdaptFramerateTimestampJitter) { + const int64_t capture_interval = VideoFormat::FpsToInterval(30); + adapter_.OnOutputFormatRequest( + VideoFormat(640, 480, capture_interval, cricket::FOURCC_ANY)); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 0 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 10 / 10 - 1, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 25 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 30 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 35 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 50 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); } // Adapt the frame rate to be half of the capture rate after capturing no less @@ -237,7 +391,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) { output_format.height *= 10; adapter_.OnOutputFormatRequest(output_format); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(capture_format_.width, cropped_width_); EXPECT_EQ(capture_format_.height, cropped_height_); @@ -250,7 +404,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) { TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) { adapter_.OnOutputFormatRequest(capture_format_); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(capture_format_.width, cropped_width_); EXPECT_EQ(capture_format_.height, cropped_height_); @@ -266,7 +420,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) { request_format.height /= 2; adapter_.OnOutputFormatRequest(request_format); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(capture_format_.width, cropped_width_); EXPECT_EQ(capture_format_.height, cropped_height_); @@ -281,7 +435,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) { output_format.height = 0; adapter_.OnOutputFormatRequest(output_format); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(0, out_width_); EXPECT_EQ(0, out_height_); @@ -348,9 +502,8 @@ TEST_F(VideoAdapterTest, DropAllFrames) { } TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), 0); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - adapter_.AdaptFrameResolution(640, 400, + VideoFormat format(640, 400, 0, 0); + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -361,7 +514,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { // Format request 640x400. format.height = 400; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -374,7 +527,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 1280; format.height = 720; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -386,7 +539,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 0; format.height = 0; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(0, out_width_); @@ -396,7 +549,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 320; format.height = 200; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -410,7 +563,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 424; format.height = 265; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -422,7 +575,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 640 * 3 / 8; format.height = 400 * 3 / 8; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -434,7 +587,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 320; format.height = 200; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -446,7 +599,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 480; format.height = 300; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -457,9 +610,8 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // Start at HD. - VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), 0); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - adapter_.AdaptFrameResolution(1280, 720, + VideoFormat format(1280, 720, 0, 0); + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -471,7 +623,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { format.width = 640; format.height = 360; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -481,7 +633,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // Now, the camera reopens at VGA. // Both the frame and the output format should be 640x360. - adapter_.AdaptFrameResolution(640, 360, + adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -492,7 +644,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // And another view request comes in for 640x360, which should have no // real impact. adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 360, + adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -503,11 +655,10 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { TEST_F(VideoAdapterTest, TestVGAWidth) { // Reqeuested Output format is 640x360. - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + VideoFormat format(640, 360, 0, FOURCC_I420); adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); // Expect cropping. @@ -517,7 +668,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { EXPECT_EQ(360, out_height_); // But if frames come in at 640x360, we shouldn't adapt them down. - adapter_.AdaptFrameResolution(640, 360, + adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -525,7 +676,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { EXPECT_EQ(640, out_width_); EXPECT_EQ(360, out_height_); - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -535,7 +686,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { } TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -546,7 +697,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt down one step. adapter_.OnResolutionRequest(rtc::Optional(1280 * 720 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -557,7 +708,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt down one step more. adapter_.OnResolutionRequest(rtc::Optional(960 * 540 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -568,7 +719,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt down one step more. adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -579,7 +730,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt up one step. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(480 * 270)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -590,7 +741,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt up one step more. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(640 * 360)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -601,7 +752,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt up one step more. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -611,7 +762,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { } TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -620,7 +771,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { EXPECT_EQ(720, out_height_); adapter_.OnResolutionRequest(rtc::Optional(0), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(0, out_width_); @@ -630,7 +781,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -640,7 +791,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -652,7 +803,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -660,10 +811,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { EXPECT_EQ(480, out_width_); EXPECT_EQ(270, out_height_); - VideoFormat new_format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + VideoFormat new_format(640, 360, 0, FOURCC_I420); adapter_.OnOutputFormatRequest(new_format); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -673,7 +823,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -683,7 +833,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { } TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -693,7 +843,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -702,7 +852,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { EXPECT_EQ(270, out_height_); adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -713,11 +863,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { // Ask for 640x360 (16:9 aspect). - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - adapter_.OnOutputFormatRequest( - VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420)); + adapter_.OnOutputFormatRequest(VideoFormat(640, 360, 0, FOURCC_I420)); // Send 640x480 (4:3 aspect). - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); // Expect cropping to 16:9 format and no scaling. @@ -730,7 +878,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); // Expect cropping to 16:9 format and 3/4 scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -742,7 +890,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(480 * 270 - 1), rtc::Optional()); // Expect cropping to 16:9 format and 1/2 scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -754,7 +902,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(320 * 180)); // Expect cropping to 16:9 format and 3/4 scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -766,7 +914,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(480 * 270)); // Expect cropping to 16:9 format and no scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -778,7 +926,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(640 * 360)); // Expect cropping to 16:9 format and no scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -789,14 +937,13 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { TEST_F(VideoAdapterTest, TestCroppingOddResolution) { // Ask for 640x360 (16:9 aspect), with 3/16 scaling. - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); adapter_.OnOutputFormatRequest( - VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420)); + VideoFormat(640, 360, 0, FOURCC_I420)); adapter_.OnResolutionRequest(rtc::Optional(640 * 360 * 3 / 16 * 3 / 16), rtc::Optional()); // Send 640x480 (4:3 aspect). - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index 084a9b913c..27da97caf4 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -227,6 +227,7 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*, if (enable_video_adapter_ && !IsScreencast()) { video_adapter_.AdaptFrameResolution( captured_frame->width, captured_frame->height, + captured_frame->time_stamp, &cropped_width, &cropped_height, &out_width, &out_height); if (out_width == 0 || out_height == 0) { diff --git a/webrtc/media/base/videocapturer.h b/webrtc/media/base/videocapturer.h index edc6cd31a1..6a57331c5a 100644 --- a/webrtc/media/base/videocapturer.h +++ b/webrtc/media/base/videocapturer.h @@ -246,11 +246,6 @@ class VideoCapturer : public sigslot::has_slots<>, void SetCaptureFormat(const VideoFormat* format) { capture_format_.reset(format ? new VideoFormat(*format) : NULL); - if (capture_format_) { - ASSERT(capture_format_->interval > 0 && - "Capture format expected to have positive interval."); - video_adapter_.SetExpectedInputFrameInterval(capture_format_->interval); - } } void SetSupportedFormats(const std::vector& formats);