diff --git a/webrtc/common_video/video_frame.cc b/webrtc/common_video/video_frame.cc index b1721b4bc4..e00ca27b8a 100644 --- a/webrtc/common_video/video_frame.cc +++ b/webrtc/common_video/video_frame.cc @@ -25,19 +25,28 @@ const size_t EncodedImage::kBufferPaddingBytesH264 = 8; VideoFrame::VideoFrame() : video_frame_buffer_(nullptr), - timestamp_(0), + timestamp_rtp_(0), ntp_time_ms_(0), - render_time_ms_(0), + timestamp_us_(0), rotation_(kVideoRotation_0) {} +VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, + webrtc::VideoRotation rotation, + int64_t timestamp_us) + : video_frame_buffer_(buffer), + timestamp_rtp_(0), + ntp_time_ms_(0), + timestamp_us_(timestamp_us), + rotation_(rotation) {} + VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, uint32_t timestamp, int64_t render_time_ms, VideoRotation rotation) : video_frame_buffer_(buffer), - timestamp_(timestamp), + timestamp_rtp_(timestamp), ntp_time_ms_(0), - render_time_ms_(render_time_ms), + timestamp_us_(render_time_ms * rtc::kNumMicrosecsPerMillisec), rotation_(rotation) { RTC_DCHECK(buffer); } @@ -55,9 +64,9 @@ void VideoFrame::CreateEmptyFrame(int width, RTC_DCHECK_GE(stride_v, half_width); // Creating empty frame - reset all values. - timestamp_ = 0; + timestamp_rtp_ = 0; ntp_time_ms_ = 0; - render_time_ms_ = 0; + timestamp_us_ = 0; rotation_ = kVideoRotation_0; // Allocate a new buffer. @@ -110,9 +119,9 @@ void VideoFrame::CopyFrame(const VideoFrame& videoFrame) { void VideoFrame::ShallowCopy(const VideoFrame& videoFrame) { video_frame_buffer_ = videoFrame.video_frame_buffer(); - timestamp_ = videoFrame.timestamp_; + timestamp_rtp_ = videoFrame.timestamp_rtp_; ntp_time_ms_ = videoFrame.ntp_time_ms_; - render_time_ms_ = videoFrame.render_time_ms_; + timestamp_us_ = videoFrame.timestamp_us_; rotation_ = videoFrame.rotation_; } diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index ea1926b388..b72c4aae74 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -1585,7 +1585,7 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream( pending_encoder_reconfiguration_(false), allocated_encoder_(nullptr, webrtc::kVideoCodecUnknown, false), sending_(false), - last_frame_timestamp_ms_(0) { + last_frame_timestamp_us_(0) { parameters_.config.rtp.max_packet_size = kVideoMtu; parameters_.conference_mode = send_params.conference_mode; @@ -1637,8 +1637,10 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::UpdateHistograms() const { void WebRtcVideoChannel2::WebRtcVideoSendStream::OnFrame( const VideoFrame& frame) { TRACE_EVENT0("webrtc", "WebRtcVideoSendStream::OnFrame"); - webrtc::VideoFrame video_frame(frame.video_frame_buffer(), 0, 0, - frame.rotation()); + webrtc::VideoFrame video_frame(frame.video_frame_buffer(), + frame.rotation(), + frame.timestamp_us()); + rtc::CritScope cs(&lock_); if (video_frame.width() != last_frame_info_.width || @@ -1662,17 +1664,7 @@ void WebRtcVideoChannel2::WebRtcVideoSendStream::OnFrame( return; } - int64_t frame_delta_ms = frame.GetTimeStamp() / rtc::kNumNanosecsPerMillisec; - - // frame->GetTimeStamp() is essentially a delta, align to webrtc time - if (!first_frame_timestamp_ms_) { - first_frame_timestamp_ms_ = - rtc::Optional(rtc::TimeMillis() - frame_delta_ms); - } - - last_frame_timestamp_ms_ = *first_frame_timestamp_ms_ + frame_delta_ms; - - video_frame.set_render_time_ms(last_frame_timestamp_ms_); + last_frame_timestamp_us_ = video_frame.timestamp_us(); if (pending_encoder_reconfiguration_) { ReconfigureEncoder(); @@ -1721,11 +1713,6 @@ bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoSend( } if (source_changing) { - // Reset timestamps to realign new incoming frames to a webrtc timestamp. - // A new source may have a different timestamp delta than the previous - // one. - first_frame_timestamp_ms_ = rtc::Optional(); - if (source == nullptr && stream_ != nullptr) { LOG(LS_VERBOSE) << "Disabling capturer, sending black frame."; // Force this black frame not to be dropped due to timestamp order @@ -1733,15 +1720,15 @@ bool WebRtcVideoChannel2::WebRtcVideoSendStream::SetVideoSend( // timestamp is less than or equal to last frame's timestamp, it is // necessary to give this black frame a larger timestamp than the // previous one. - last_frame_timestamp_ms_ += 1; + last_frame_timestamp_us_ += rtc::kNumMicrosecsPerMillisec; rtc::scoped_refptr black_buffer( webrtc::I420Buffer::Create(last_frame_info_.width, last_frame_info_.height)); black_buffer->SetToBlack(); stream_->Input()->IncomingCapturedFrame(webrtc::VideoFrame( - black_buffer, 0 /* timestamp (90 kHz) */, - last_frame_timestamp_ms_, last_frame_info_.rotation)); + black_buffer, last_frame_info_.rotation, + last_frame_timestamp_us_)); } source_ = source; } diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h index 4f8fcd8337..7e32116608 100644 --- a/webrtc/media/engine/webrtcvideoengine2.h +++ b/webrtc/media/engine/webrtcvideoengine2.h @@ -406,13 +406,9 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, public webrtc::Transport { bool sending_ GUARDED_BY(lock_); - // The timestamp of the first frame received - // Used to generate the timestamps of subsequent frames - rtc::Optional first_frame_timestamp_ms_ GUARDED_BY(lock_); - // The timestamp of the last frame received // Used to generate timestamp for the black frame when source is removed - int64_t last_frame_timestamp_ms_ GUARDED_BY(lock_); + int64_t last_frame_timestamp_us_ GUARDED_BY(lock_); }; // Wrapper for the receiver part, contains configs etc. that are needed to diff --git a/webrtc/media/engine/webrtcvideoengine2_unittest.cc b/webrtc/media/engine/webrtcvideoengine2_unittest.cc index 05f5bf38d9..260fca6eaa 100644 --- a/webrtc/media/engine/webrtcvideoengine2_unittest.cc +++ b/webrtc/media/engine/webrtcvideoengine2_unittest.cc @@ -520,63 +520,6 @@ TEST_F(WebRtcVideoEngine2Test, PropagatesInputFrameTimestamp) { EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); } -TEST_F(WebRtcVideoEngine2Test, - ProducesIncreasingTimestampsWithResetInputSources) { - cricket::FakeWebRtcVideoEncoderFactory encoder_factory; - encoder_factory.AddSupportedVideoCodecType(webrtc::kVideoCodecVP8, "VP8"); - std::vector codecs; - codecs.push_back(kVp8Codec); - - FakeCall* fake_call = new FakeCall(webrtc::Call::Config()); - call_.reset(fake_call); - std::unique_ptr channel( - SetUpForExternalEncoderFactory(&encoder_factory, codecs)); - - EXPECT_TRUE( - channel->AddSendStream(cricket::StreamParams::CreateLegacy(kSsrc))); - channel->SetSend(true); - FakeVideoSendStream* stream = fake_call->GetVideoSendStreams()[0]; - - FakeVideoCapturer capturer1; - EXPECT_TRUE(channel->SetVideoSend(kSsrc, true, nullptr, &capturer1)); - - cricket::CapturedFrame frame; - frame.width = 1280; - frame.height = 720; - frame.fourcc = cricket::FOURCC_I420; - frame.data_size = frame.width * frame.height + - 2 * ((frame.width + 1) / 2) * ((frame.height + 1) / 2); - std::unique_ptr data(new char[frame.data_size]); - frame.data = data.get(); - memset(frame.data, 1, frame.data_size); - int64_t initial_timestamp = rtc::TimeNanos(); - frame.time_stamp = initial_timestamp; - - // Deliver initial frame. - capturer1.SignalCapturedFrame(&frame); - // Deliver next frame 1 second later. - frame.time_stamp += rtc::kNumNanosecsPerSec; - rtc::Thread::Current()->SleepMs(1000); - capturer1.SignalCapturedFrame(&frame); - - int64_t capturer1_last_timestamp = stream->GetLastTimestamp(); - // Reset input source, should still be continuous even though input-frame - // timestamp is less than before. - FakeVideoCapturer capturer2; - EXPECT_TRUE(channel->SetVideoSend(kSsrc, true, nullptr, &capturer2)); - - rtc::Thread::Current()->SleepMs(1); - // Deliver with a timestamp (10 seconds) before the previous initial one, - // these should not be related at all anymore and it should still work fine. - frame.time_stamp = initial_timestamp - 10 * rtc::kNumNanosecsPerSec; - capturer2.SignalCapturedFrame(&frame); - - // New timestamp should be at least 1ms in the future and not old. - EXPECT_GT(stream->GetLastTimestamp(), capturer1_last_timestamp); - - EXPECT_TRUE(channel->RemoveSendStream(kSsrc)); -} - VideoMediaChannel* WebRtcVideoEngine2Test::SetUpForExternalEncoderFactory( cricket::WebRtcVideoEncoderFactory* encoder_factory, const std::vector& codecs) { diff --git a/webrtc/video_frame.h b/webrtc/video_frame.h index ab8e37b0ed..b99e60d621 100644 --- a/webrtc/video_frame.h +++ b/webrtc/video_frame.h @@ -12,6 +12,7 @@ #define WEBRTC_VIDEO_FRAME_H_ #include "webrtc/base/scoped_ref_ptr.h" +#include "webrtc/base/timeutils.h" #include "webrtc/common_types.h" #include "webrtc/common_video/include/video_frame_buffer.h" #include "webrtc/common_video/rotation.h" @@ -28,8 +29,17 @@ class VideoFrame { // reasonable assumption that video_frame_buffer() returns a valid buffer. VideoFrame(); + // TODO(nisse): This constructor is consistent with + // cricket::WebRtcVideoFrame. After the class + // cricket::WebRtcVideoFrame and its baseclass cricket::VideoFrame + // are deleted, we should consider whether or not we want to stick + // to this style and deprecate the other constructors. + VideoFrame(const rtc::scoped_refptr& buffer, + webrtc::VideoRotation rotation, + int64_t timestamp_us); + // Preferred constructor. - VideoFrame(const rtc::scoped_refptr& buffer, + VideoFrame(const rtc::scoped_refptr& buffer, uint32_t timestamp, int64_t render_time_ms, VideoRotation rotation); @@ -103,15 +113,21 @@ class VideoFrame { // Get frame height. int height() const; + // System monotonic clock, same timebase as rtc::TimeMicros(). + int64_t timestamp_us() const { return timestamp_us_; } + void set_timestamp_us(int64_t timestamp_us) { + timestamp_us_ = timestamp_us; + } + // TODO(nisse): After the cricket::VideoFrame and webrtc::VideoFrame - // merge, we'll have methods timestamp_us and set_timestamp_us, all - // other frame timestamps will likely be deprecated. + // merge, timestamps other than timestamp_us will likely be + // deprecated. // Set frame timestamp (90kHz). - void set_timestamp(uint32_t timestamp) { timestamp_ = timestamp; } + void set_timestamp(uint32_t timestamp) { timestamp_rtp_ = timestamp; } // Get frame timestamp (90kHz). - uint32_t timestamp() const { return timestamp_; } + uint32_t timestamp() const { return timestamp_rtp_; } // Set capture ntp time in milliseconds. void set_ntp_time_ms(int64_t ntp_time_ms) { @@ -138,11 +154,13 @@ class VideoFrame { // Set render time in milliseconds. void set_render_time_ms(int64_t render_time_ms) { - render_time_ms_ = render_time_ms; + set_timestamp_us(render_time_ms * rtc::kNumMicrosecsPerMillisec);; } // Get render time in milliseconds. - int64_t render_time_ms() const { return render_time_ms_; } + int64_t render_time_ms() const { + return timestamp_us() / rtc::kNumMicrosecsPerMillisec; + } // Return true if and only if video_frame_buffer() is null. Which is possible // only if the object was default-constructed. @@ -169,9 +187,9 @@ class VideoFrame { private: // An opaque reference counted handle that stores the pixel data. rtc::scoped_refptr video_frame_buffer_; - uint32_t timestamp_; + uint32_t timestamp_rtp_; int64_t ntp_time_ms_; - int64_t render_time_ms_; + int64_t timestamp_us_; VideoRotation rotation_; };