diff --git a/webrtc/media/base/videoadapter.cc b/webrtc/media/base/videoadapter.cc index 9c3837c572..e1f506f10b 100644 --- a/webrtc/media/base/videoadapter.cc +++ b/webrtc/media/base/videoadapter.cc @@ -11,6 +11,7 @@ #include "webrtc/media/base/videoadapter.h" #include +#include #include #include "webrtc/base/checks.h" @@ -118,22 +119,43 @@ VideoAdapter::VideoAdapter() adaption_changes_(0), previous_width_(0), previous_height_(0), - input_interval_(0), - interval_next_frame_(0), resolution_request_max_pixel_count_(std::numeric_limits::max()), resolution_request_max_pixel_count_step_up_(0) {} VideoAdapter::~VideoAdapter() {} -void VideoAdapter::SetExpectedInputFrameInterval(int64_t interval) { - // TODO(perkj): Consider measuring input frame rate instead. - // Frame rate typically varies depending on lighting. +bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) { rtc::CritScope cs(&critical_section_); - input_interval_ = interval; + if (!requested_format_ || requested_format_->interval == 0) + return true; + + if (next_frame_timestamp_ns_) { + // Time until next frame should be outputted. + const int64_t time_until_next_frame_ns = + (*next_frame_timestamp_ns_ - in_timestamp_ns); + + // Continue if timestamp is withing expected range. + if (std::abs(time_until_next_frame_ns) < 2 * requested_format_->interval) { + // Drop if a frame shouldn't be outputted yet. + if (time_until_next_frame_ns > 0) + return false; + // Time to output new frame. + *next_frame_timestamp_ns_ += requested_format_->interval; + return true; + } + } + + // First timestamp received or timestamp is way outside expected range, so + // reset. Set first timestamp target to just half the interval to prefer + // keeping frames in case of jitter. + next_frame_timestamp_ns_ = + rtc::Optional(in_timestamp_ns + requested_format_->interval / 2); + return true; } void VideoAdapter::AdaptFrameResolution(int in_width, int in_height, + int64_t in_timestamp_ns, int* cropped_width, int* cropped_height, int* out_width, @@ -150,25 +172,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, } // Drop the input frame if necessary. - bool should_drop = false; - if (max_pixel_count == 0) { - // Drop all frames as the output format is 0x0. - should_drop = true; - } else if (requested_format_ && requested_format_->interval > 0) { - // Drop some frames based on input fps and output fps. - // Normally output fps is less than input fps. - interval_next_frame_ += input_interval_; - if (interval_next_frame_ >= requested_format_->interval) { - interval_next_frame_ -= requested_format_->interval; - // Reset |interval_next_frame_| if it accumulates too much to avoid - // "catching up" behaviour. - if (interval_next_frame_ >= requested_format_->interval) - interval_next_frame_ = 0; - } else { - should_drop = true; - } - } - if (should_drop) { + if (max_pixel_count == 0 || !KeepFrame(in_timestamp_ns)) { // Show VAdapt log every 90 frames dropped. (3 seconds) if ((frames_in_ - frames_out_) % 90 == 0) { // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed @@ -179,7 +183,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, << " Changes: " << adaption_changes_ << " Input: " << in_width << "x" << in_height - << " i" << input_interval_ + << " timestamp: " << in_timestamp_ns << " Output: i" << (requested_format_ ? requested_format_->interval : 0); } @@ -238,7 +242,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out " << frames_out_ << " / in " << frames_in_ << " Changes: " << adaption_changes_ << " Input: " << in_width - << "x" << in_height << " i" << input_interval_ + << "x" << in_height << " Scale: " << scale.numerator << "/" << scale.denominator << " Output: " << *out_width << "x" << *out_height << " i" << (requested_format_ ? requested_format_->interval : 0); @@ -251,7 +255,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width, void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) { rtc::CritScope cs(&critical_section_); requested_format_ = rtc::Optional(format); - interval_next_frame_ = 0; + next_frame_timestamp_ns_ = rtc::Optional(); } void VideoAdapter::OnResolutionRequest( diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h index 2db0ada271..45fd1fa827 100644 --- a/webrtc/media/base/videoadapter.h +++ b/webrtc/media/base/videoadapter.h @@ -27,17 +27,13 @@ class VideoAdapter { VideoAdapter(); virtual ~VideoAdapter(); - // Sets the expected frame interval. This controls how often frames should - // be dropped if |OnOutputFormatRequest| is called with a lower frame - // interval. - void SetExpectedInputFrameInterval(int64_t interval); - // Return the adapted resolution given the input resolution. The input // resolution should first be cropped to the specified resolution, and then // scaled to the final output resolution. The output resolution will be 0x0 if // the frame should be dropped. void AdaptFrameResolution(int in_width, int in_height, + int64_t in_timestamp_ns, int* cropped_width, int* cropped_height, int* out_width, @@ -58,14 +54,17 @@ class VideoAdapter { rtc::Optional max_pixel_count_step_up); private: + // Determine if frame should be dropped based on input fps and requested fps. + bool KeepFrame(int64_t in_timestamp_ns); + int frames_in_; // Number of input frames. int frames_out_; // Number of output frames. int frames_scaled_; // Number of frames scaled. int adaption_changes_; // Number of changes in scale factor. int previous_width_; // Previous adapter output width. int previous_height_; // Previous adapter output height. - int input_interval_ GUARDED_BY(critical_section_); - int64_t interval_next_frame_ GUARDED_BY(critical_section_); + // The target timestamp for the next frame based on requested format. + rtc::Optional next_frame_timestamp_ns_ GUARDED_BY(critical_section_); // Max number of pixels requested via calls to OnOutputFormatRequest, // OnResolutionRequest respectively. diff --git a/webrtc/media/base/videoadapter_unittest.cc b/webrtc/media/base/videoadapter_unittest.cc index 92b4a558b9..d76bbd6ac8 100644 --- a/webrtc/media/base/videoadapter_unittest.cc +++ b/webrtc/media/base/videoadapter_unittest.cc @@ -28,8 +28,7 @@ class VideoAdapterTest : public testing::Test { virtual void SetUp() { capturer_.reset(new FakeVideoCapturer); capture_format_ = capturer_->GetSupportedFormats()->at(0); - capture_format_.interval = VideoFormat::FpsToInterval(50); - adapter_.SetExpectedInputFrameInterval(capture_format_.interval); + capture_format_.interval = VideoFormat::FpsToInterval(30); listener_.reset(new VideoCapturerListener(&adapter_)); capturer_->SignalFrameCaptured.connect( @@ -73,6 +72,7 @@ class VideoAdapterTest : public testing::Test { int out_width; int out_height; video_adapter_->AdaptFrameResolution(in_width, in_height, + captured_frame->time_stamp, &cropped_width, &cropped_height, &out_width, &out_height); if (out_width != 0 && out_height != 0) { @@ -170,39 +170,193 @@ TEST_F(VideoAdapterTest, AdaptZeroInterval) { // Adapt the frame rate to be half of the capture rate at the beginning. Expect // the number of dropped frames to be half of the number the captured frames. -TEST_F(VideoAdapterTest, AdaptFramerate) { +TEST_F(VideoAdapterTest, AdaptFramerateToHalf) { VideoFormat request_format = capture_format_; request_format.interval *= 2; adapter_.OnOutputFormatRequest(request_format); EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); + + // Capture 10 frames and verify that every other frame is dropped. The first + // frame should not be dropped. + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 1); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 2); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 3); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 4); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 5); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 6); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 7); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 8); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 9); + EXPECT_EQ(4, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 10); + EXPECT_EQ(4, listener_->GetStats().dropped_frames); +} + +// Adapt the frame rate to be two thirds of the capture rate at the beginning. +// Expect the number of dropped frames to be one thirds of the number the +// captured frames. +TEST_F(VideoAdapterTest, AdaptFramerateToTwoThirds) { + VideoFormat request_format = capture_format_; + request_format.interval = request_format.interval * 3 / 2; + adapter_.OnOutputFormatRequest(request_format); + EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); + + // Capture 10 frames and verify that every third frame is dropped. The first + // frame should not be dropped. + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 1); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 2); + EXPECT_EQ(0, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 3); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 4); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 5); + EXPECT_EQ(1, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 6); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 7); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 8); + EXPECT_EQ(2, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 9); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); + + capturer_->CaptureFrame(); + EXPECT_GE(listener_->GetStats().captured_frames, 10); + EXPECT_EQ(3, listener_->GetStats().dropped_frames); +} + +// Request frame rate twice as high as captured frame rate. Expect no frame +// drop. +TEST_F(VideoAdapterTest, AdaptFramerateHighLimit) { + VideoFormat request_format = capture_format_; + request_format.interval /= 2; + adapter_.OnOutputFormatRequest(request_format); + EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); for (int i = 0; i < 10; ++i) capturer_->CaptureFrame(); - // Verify frame drop and no resolution change. - VideoCapturerListener::Stats stats = listener_->GetStats(); - EXPECT_GE(stats.captured_frames, 10); - EXPECT_EQ(stats.captured_frames / 2, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, - capture_format_.width, capture_format_.height); + // Verify no frame drop. + EXPECT_EQ(0, listener_->GetStats().dropped_frames); } -// Adapt the frame rate to be half of the capture rate at the beginning. Expect -// the number of dropped frames to be half of the number the captured frames. -TEST_F(VideoAdapterTest, AdaptFramerateVariable) { - VideoFormat request_format = capture_format_; - request_format.interval = request_format.interval * 3 / 2; - adapter_.OnOutputFormatRequest(request_format); - EXPECT_EQ(CS_RUNNING, capturer_->Start(capture_format_)); - for (int i = 0; i < 30; ++i) - capturer_->CaptureFrame(); +// After the first timestamp, add a big offset to the timestamps. Expect that +// the adapter is conservative and resets to the new offset and does not drop +// any frame. +TEST_F(VideoAdapterTest, AdaptFramerateTimestampOffset) { + const int64_t capture_interval = VideoFormat::FpsToInterval(30); + adapter_.OnOutputFormatRequest( + VideoFormat(640, 480, capture_interval, cricket::FOURCC_ANY)); - // Verify frame drop and no resolution change. - VideoCapturerListener::Stats stats = listener_->GetStats(); - EXPECT_GE(stats.captured_frames, 30); - // Verify 2 / 3 kept (20) and 1 / 3 dropped (10). - EXPECT_EQ(stats.captured_frames * 1 / 3, stats.dropped_frames); - VerifyAdaptedResolution(stats, capture_format_.width, capture_format_.height, - capture_format_.width, capture_format_.height); + const int64_t first_timestamp = 0; + adapter_.AdaptFrameResolution(640, 480, first_timestamp, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + const int64_t big_offset = -987654321LL * 1000; + const int64_t second_timestamp = big_offset; + adapter_.AdaptFrameResolution(640, 480, second_timestamp, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + const int64_t third_timestamp = big_offset + capture_interval; + adapter_.AdaptFrameResolution(640, 480, third_timestamp, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); +} + +// Request 30 fps and send 30 fps with jitter. Expect that no frame is dropped. +TEST_F(VideoAdapterTest, AdaptFramerateTimestampJitter) { + const int64_t capture_interval = VideoFormat::FpsToInterval(30); + adapter_.OnOutputFormatRequest( + VideoFormat(640, 480, capture_interval, cricket::FOURCC_ANY)); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 0 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 10 / 10 - 1, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 25 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 30 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 35 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); + + adapter_.AdaptFrameResolution(640, 480, capture_interval * 50 / 10, + &cropped_width_, &cropped_height_, + &out_width_, &out_height_); + EXPECT_GT(out_width_, 0); + EXPECT_GT(out_height_, 0); } // Adapt the frame rate to be half of the capture rate after capturing no less @@ -237,7 +391,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) { output_format.height *= 10; adapter_.OnOutputFormatRequest(output_format); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(capture_format_.width, cropped_width_); EXPECT_EQ(capture_format_.height, cropped_height_); @@ -250,7 +404,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) { TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) { adapter_.OnOutputFormatRequest(capture_format_); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(capture_format_.width, cropped_width_); EXPECT_EQ(capture_format_.height, cropped_height_); @@ -266,7 +420,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) { request_format.height /= 2; adapter_.OnOutputFormatRequest(request_format); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(capture_format_.width, cropped_width_); EXPECT_EQ(capture_format_.height, cropped_height_); @@ -281,7 +435,7 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) { output_format.height = 0; adapter_.OnOutputFormatRequest(output_format); adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height, - &cropped_width_, &cropped_height_, + 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(0, out_width_); EXPECT_EQ(0, out_height_); @@ -348,9 +502,8 @@ TEST_F(VideoAdapterTest, DropAllFrames) { } TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { - VideoFormat format(640, 400, VideoFormat::FpsToInterval(30), 0); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - adapter_.AdaptFrameResolution(640, 400, + VideoFormat format(640, 400, 0, 0); + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -361,7 +514,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { // Format request 640x400. format.height = 400; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -374,7 +527,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 1280; format.height = 720; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -386,7 +539,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 0; format.height = 0; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(0, out_width_); @@ -396,7 +549,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 320; format.height = 200; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -410,7 +563,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 424; format.height = 265; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -422,7 +575,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 640 * 3 / 8; format.height = 400 * 3 / 8; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -434,7 +587,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 320; format.height = 200; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -446,7 +599,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { format.width = 480; format.height = 300; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 400, + adapter_.AdaptFrameResolution(640, 400, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -457,9 +610,8 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) { TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // Start at HD. - VideoFormat format(1280, 720, VideoFormat::FpsToInterval(30), 0); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - adapter_.AdaptFrameResolution(1280, 720, + VideoFormat format(1280, 720, 0, 0); + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -471,7 +623,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { format.width = 640; format.height = 360; adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -481,7 +633,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // Now, the camera reopens at VGA. // Both the frame and the output format should be 640x360. - adapter_.AdaptFrameResolution(640, 360, + adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -492,7 +644,7 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { // And another view request comes in for 640x360, which should have no // real impact. adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 360, + adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -503,11 +655,10 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) { TEST_F(VideoAdapterTest, TestVGAWidth) { // Reqeuested Output format is 640x360. - VideoFormat format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + VideoFormat format(640, 360, 0, FOURCC_I420); adapter_.OnOutputFormatRequest(format); - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); // Expect cropping. @@ -517,7 +668,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { EXPECT_EQ(360, out_height_); // But if frames come in at 640x360, we shouldn't adapt them down. - adapter_.AdaptFrameResolution(640, 360, + adapter_.AdaptFrameResolution(640, 360, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -525,7 +676,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { EXPECT_EQ(640, out_width_); EXPECT_EQ(360, out_height_); - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -535,7 +686,7 @@ TEST_F(VideoAdapterTest, TestVGAWidth) { } TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -546,7 +697,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt down one step. adapter_.OnResolutionRequest(rtc::Optional(1280 * 720 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -557,7 +708,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt down one step more. adapter_.OnResolutionRequest(rtc::Optional(960 * 540 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -568,7 +719,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt down one step more. adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -579,7 +730,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt up one step. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(480 * 270)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -590,7 +741,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt up one step more. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(640 * 360)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -601,7 +752,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { // Adapt up one step more. adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -611,7 +762,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) { } TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -620,7 +771,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { EXPECT_EQ(720, out_height_); adapter_.OnResolutionRequest(rtc::Optional(0), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(0, out_width_); @@ -630,7 +781,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) { TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -640,7 +791,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -652,7 +803,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) { TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -660,10 +811,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { EXPECT_EQ(480, out_width_); EXPECT_EQ(270, out_height_); - VideoFormat new_format(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420); - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); + VideoFormat new_format(640, 360, 0, FOURCC_I420); adapter_.OnOutputFormatRequest(new_format); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -673,7 +823,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(960 * 720)); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -683,7 +833,7 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) { } TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -693,7 +843,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -702,7 +852,7 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { EXPECT_EQ(270, out_height_); adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional()); - adapter_.AdaptFrameResolution(1280, 720, + adapter_.AdaptFrameResolution(1280, 720, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(1280, cropped_width_); @@ -713,11 +863,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) { TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { // Ask for 640x360 (16:9 aspect). - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); - adapter_.OnOutputFormatRequest( - VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420)); + adapter_.OnOutputFormatRequest(VideoFormat(640, 360, 0, FOURCC_I420)); // Send 640x480 (4:3 aspect). - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); // Expect cropping to 16:9 format and no scaling. @@ -730,7 +878,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(640 * 360 - 1), rtc::Optional()); // Expect cropping to 16:9 format and 3/4 scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -742,7 +890,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(480 * 270 - 1), rtc::Optional()); // Expect cropping to 16:9 format and 1/2 scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -754,7 +902,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(320 * 180)); // Expect cropping to 16:9 format and 3/4 scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -766,7 +914,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(480 * 270)); // Expect cropping to 16:9 format and no scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -778,7 +926,7 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { adapter_.OnResolutionRequest(rtc::Optional(), rtc::Optional(640 * 360)); // Expect cropping to 16:9 format and no scaling. - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); EXPECT_EQ(640, cropped_width_); @@ -789,14 +937,13 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) { TEST_F(VideoAdapterTest, TestCroppingOddResolution) { // Ask for 640x360 (16:9 aspect), with 3/16 scaling. - adapter_.SetExpectedInputFrameInterval(VideoFormat::FpsToInterval(30)); adapter_.OnOutputFormatRequest( - VideoFormat(640, 360, VideoFormat::FpsToInterval(30), FOURCC_I420)); + VideoFormat(640, 360, 0, FOURCC_I420)); adapter_.OnResolutionRequest(rtc::Optional(640 * 360 * 3 / 16 * 3 / 16), rtc::Optional()); // Send 640x480 (4:3 aspect). - adapter_.AdaptFrameResolution(640, 480, + adapter_.AdaptFrameResolution(640, 480, 0, &cropped_width_, &cropped_height_, &out_width_, &out_height_); diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc index 084a9b913c..27da97caf4 100644 --- a/webrtc/media/base/videocapturer.cc +++ b/webrtc/media/base/videocapturer.cc @@ -227,6 +227,7 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*, if (enable_video_adapter_ && !IsScreencast()) { video_adapter_.AdaptFrameResolution( captured_frame->width, captured_frame->height, + captured_frame->time_stamp, &cropped_width, &cropped_height, &out_width, &out_height); if (out_width == 0 || out_height == 0) { diff --git a/webrtc/media/base/videocapturer.h b/webrtc/media/base/videocapturer.h index edc6cd31a1..6a57331c5a 100644 --- a/webrtc/media/base/videocapturer.h +++ b/webrtc/media/base/videocapturer.h @@ -246,11 +246,6 @@ class VideoCapturer : public sigslot::has_slots<>, void SetCaptureFormat(const VideoFormat* format) { capture_format_.reset(format ? new VideoFormat(*format) : NULL); - if (capture_format_) { - ASSERT(capture_format_->interval > 0 && - "Capture format expected to have positive interval."); - video_adapter_.SetExpectedInputFrameInterval(capture_format_->interval); - } } void SetSupportedFormats(const std::vector& formats);