diff --git a/talk/media/base/fakevideocapturer.h b/talk/media/base/fakevideocapturer.h index c9dce2c1d9..ef6288ac39 100644 --- a/talk/media/base/fakevideocapturer.h +++ b/talk/media/base/fakevideocapturer.h @@ -64,6 +64,8 @@ class FakeVideoCapturer : public cricket::VideoCapturer { cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420)); formats.push_back(cricket::VideoFormat(160, 120, cricket::VideoFormat::FpsToInterval(30), cricket::FOURCC_I420)); + formats.push_back(cricket::VideoFormat(1280, 720, + cricket::VideoFormat::FpsToInterval(60), cricket::FOURCC_I420)); ResetSupportedFormats(formats); } ~FakeVideoCapturer() { @@ -79,9 +81,17 @@ class FakeVideoCapturer : public cricket::VideoCapturer { } return CaptureCustomFrame(GetCaptureFormat()->width, GetCaptureFormat()->height, + GetCaptureFormat()->interval, GetCaptureFormat()->fourcc); } bool CaptureCustomFrame(int width, int height, uint32 fourcc) { + // default to 30fps + return CaptureCustomFrame(width, height, 33333333, fourcc); + } + bool CaptureCustomFrame(int width, + int height, + int64_t timestamp_interval, + uint32 fourcc) { if (!running_) { return false; } @@ -106,7 +116,7 @@ class FakeVideoCapturer : public cricket::VideoCapturer { frame.data_size = size; frame.elapsed_time = next_timestamp_; frame.time_stamp = initial_unix_timestamp_ + next_timestamp_; - next_timestamp_ += 33333333; // 30 fps + next_timestamp_ += timestamp_interval; rtc::scoped_ptr data(new char[size]); frame.data = data.get(); diff --git a/talk/media/webrtc/fakewebrtccall.cc b/talk/media/webrtc/fakewebrtccall.cc index 03f7763bb2..742c970909 100644 --- a/talk/media/webrtc/fakewebrtccall.cc +++ b/talk/media/webrtc/fakewebrtccall.cc @@ -112,6 +112,11 @@ int FakeVideoSendStream::GetLastHeight() const { return last_frame_.height(); } +int64_t FakeVideoSendStream::GetLastTimestamp() const { + DCHECK(last_frame_.ntp_time_ms() == 0); + return last_frame_.render_time_ms(); +} + void FakeVideoSendStream::IncomingCapturedFrame( const webrtc::VideoFrame& frame) { ++num_swapped_frames_; diff --git a/talk/media/webrtc/fakewebrtccall.h b/talk/media/webrtc/fakewebrtccall.h index dd7772d222..5f217822ef 100644 --- a/talk/media/webrtc/fakewebrtccall.h +++ b/talk/media/webrtc/fakewebrtccall.h @@ -82,6 +82,7 @@ class FakeVideoSendStream : public webrtc::VideoSendStream, int GetNumberOfSwappedFrames() const; int GetLastWidth() const; int GetLastHeight() const; + int64_t GetLastTimestamp() const; void SetStats(const webrtc::VideoSendStream::Stats& stats); private: diff --git a/talk/media/webrtc/webrtcvideoengine2.cc b/talk/media/webrtc/webrtcvideoengine2.cc index 91997954d6..3fd5690125 100644 --- a/talk/media/webrtc/webrtcvideoengine2.cc +++ b/talk/media/webrtc/webrtcvideoengine2.cc @@ -42,6 +42,7 @@ #include "webrtc/base/buffer.h" #include "webrtc/base/logging.h" #include "webrtc/base/stringutils.h" +#include "webrtc/base/timeutils.h" #include "webrtc/call.h" #include "webrtc/modules/video_coding/codecs/h264/include/h264.h" #include "webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h" @@ -1670,7 +1671,9 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( capturer_(NULL), sending_(false), muted_(false), - old_adapt_changes_(0) { + old_adapt_changes_(0), + first_frame_timestamp_ms_(0), + last_frame_timestamp_ms_(0) { parameters_.config.rtp.max_packet_size = kVideoMtu; sp.GetPrimarySsrcs(¶meters_.config.rtp.ssrcs); @@ -1734,6 +1737,15 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::InputFrame( static_cast(frame->GetWidth()), static_cast(frame->GetHeight())); } + + int64_t frame_delta_ms = frame->GetTimeStamp() / rtc::kNumNanosecsPerMillisec; + // frame->GetTimeStamp() is essentially a delta, align to webrtc time + if (first_frame_timestamp_ms_ == 0) { + first_frame_timestamp_ms_ = rtc::Time() - frame_delta_ms; + } + + last_frame_timestamp_ms_ = first_frame_timestamp_ms_ + frame_delta_ms; + video_frame.set_render_time_ms(last_frame_timestamp_ms_); // Reconfigure codec if necessary. SetDimensions( video_frame.width(), video_frame.height(), capturer->IsScreencast()); @@ -1762,6 +1774,15 @@ bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetCapturer( CreateBlackFrame(&black_frame, last_dimensions_.width, last_dimensions_.height); + + // Force this black frame not to be dropped due to timestamp order + // check. As IncomingCapturedFrame will drop the frame if this frame's + // timestamp is less than or equal to last frame's timestamp, it is + // necessary to give this black frame a larger timestamp than the + // previous one. + last_frame_timestamp_ms_ += + format_.interval / rtc::kNumNanosecsPerMillisec; + black_frame.set_render_time_ms(last_frame_timestamp_ms_); stream_->Input()->IncomingCapturedFrame(black_frame); } diff --git a/talk/media/webrtc/webrtcvideoengine2.h b/talk/media/webrtc/webrtcvideoengine2.h index e305975840..2cbebf81d6 100644 --- a/talk/media/webrtc/webrtcvideoengine2.h +++ b/talk/media/webrtc/webrtcvideoengine2.h @@ -393,6 +393,14 @@ class WebRtcVideoChannel2 : public rtc::MessageHandler, bool muted_ GUARDED_BY(lock_); VideoFormat format_ GUARDED_BY(lock_); int old_adapt_changes_ GUARDED_BY(lock_); + + // The timestamp of the first frame received + // Used to generate the timestamps of subsequent frames + int64_t first_frame_timestamp_ms_ GUARDED_BY(lock_); + + // The timestamp of the last frame received + // Used to generate timestamp for the black frame when capturer is removed + int64_t last_frame_timestamp_ms_ GUARDED_BY(lock_); }; // Wrapper for the receiver part, contains configs etc. that are needed to diff --git a/talk/media/webrtc/webrtcvideoengine2_unittest.cc b/talk/media/webrtc/webrtcvideoengine2_unittest.cc index 8875994fb8..5c4fd6265e 100644 --- a/talk/media/webrtc/webrtcvideoengine2_unittest.cc +++ b/talk/media/webrtc/webrtcvideoengine2_unittest.cc @@ -419,6 +419,70 @@ TEST_F(WebRtcVideoEngine2Test, CanConstructDecoderForVp9EncoderFactory) { channel->AddRecvStream(cricket::StreamParams::CreateLegacy(kSsrc))); } +TEST_F(WebRtcVideoEngine2Test, PropagatesInputFrameTimestamp) { + cricket::FakeWebRtcVideoEncoderFactory encoder_factory; + encoder_factory.AddSupportedVideoCodecType(webrtc::kVideoCodecVP8, "VP8"); + std::vector codecs; + codecs.push_back(kVp8Codec); + + FakeCallFactory factory; + engine_.SetCallFactory(&factory); + rtc::scoped_ptr channel( + SetUpForExternalEncoderFactory(&encoder_factory, codecs)); + + EXPECT_TRUE( + channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); + + FakeVideoCapturer capturer; + EXPECT_TRUE(channel->SetCapturer(kSsrc, &capturer)); + capturer.Start(cricket::VideoFormat(1280, 720, + cricket::VideoFormat::FpsToInterval(60), + cricket::FOURCC_I420)); + channel->SetSend(true); + + FakeCall* call = factory.GetCall(); + std::vector streams = call->GetVideoSendStreams(); + FakeVideoSendStream* stream = streams[0]; + + int64_t timestamp; + int64_t last_timestamp; + + EXPECT_TRUE(capturer.CaptureFrame()); + last_timestamp = stream->GetLastTimestamp(); + for (int i = 0; i < 10; i++) { + EXPECT_TRUE(capturer.CaptureFrame()); + timestamp = stream->GetLastTimestamp(); + int64_t interval = timestamp - last_timestamp; + + // Precision changes from nanosecond to millisecond. + // Allow error to be no more than 1. + EXPECT_NEAR(cricket::VideoFormat::FpsToInterval(60) / 1E6, interval, 1); + + last_timestamp = timestamp; + } + + capturer.Start(cricket::VideoFormat(1280, 720, + cricket::VideoFormat::FpsToInterval(30), + cricket::FOURCC_I420)); + + EXPECT_TRUE(capturer.CaptureFrame()); + last_timestamp = stream->GetLastTimestamp(); + for (int i = 0; i < 10; i++) { + EXPECT_TRUE(capturer.CaptureFrame()); + timestamp = stream->GetLastTimestamp(); + int64_t interval = timestamp - last_timestamp; + + // Precision changes from nanosecond to millisecond. + // Allow error to be no more than 1. + EXPECT_NEAR(cricket::VideoFormat::FpsToInterval(30) / 1E6, interval, 1); + + last_timestamp = timestamp; + } + + // Remove stream previously added to free the external encoder instance. + EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); +} + VideoMediaChannel* WebRtcVideoEngine2Test::SetUpForExternalEncoderFactory( cricket::WebRtcVideoEncoderFactory* encoder_factory, const std::vector& codecs) {