From 19ee1e6eb102b2282a9993d6c373c7185ad9ba34 Mon Sep 17 00:00:00 2001 From: Sergey Ulanov Date: Mon, 1 Aug 2016 13:35:55 -0700 Subject: [PATCH] Add cricket::VideoFrame::transport_frame_id() and set it to RTP timestamp. Passing transport_frame_id() to VideoSink will allow to identify incoming video frames, which will make it possible to correlate video frames on the sender and on the receiver. BUG=chromium:621691 R=mflodman@webrtc.org, stefan@webrtc.org Review URL: https://codereview.webrtc.org/2088953002 . Cr-Commit-Position: refs/heads/master@{#13596} --- .../android/jni/androidvideocapturer_jni.cc | 33 +++++++++---------- webrtc/api/androidvideotracksource.cc | 4 +-- webrtc/media/base/videobroadcaster.cc | 4 +-- .../media/base/videobroadcaster_unittest.cc | 8 ++--- webrtc/media/base/videoframe.h | 3 ++ webrtc/media/base/videoframefactory.cc | 3 +- webrtc/media/engine/webrtcvideoengine2.cc | 4 +-- webrtc/media/engine/webrtcvideoframe.cc | 30 ++++++++++++++--- webrtc/media/engine/webrtcvideoframe.h | 17 ++++++---- .../Classes/RTCVideoRendererAdapter.mm | 6 ++-- .../Classes/avfoundationvideocapturer.mm | 2 +- 11 files changed, 70 insertions(+), 44 deletions(-) diff --git a/webrtc/api/android/jni/androidvideocapturer_jni.cc b/webrtc/api/android/jni/androidvideocapturer_jni.cc index f9c334b394..d31ce26aab 100644 --- a/webrtc/api/android/jni/androidvideocapturer_jni.cc +++ b/webrtc/api/android/jni/androidvideocapturer_jni.cc @@ -234,13 +234,13 @@ void AndroidVideoCapturerJni::OnMemoryBufferFrame(void* video_frame, scaled_buffer->ScaleFrom(buffer); buffer = scaled_buffer; } - capturer_->OnFrame(cricket::WebRtcVideoFrame( - buffer, - capturer_->apply_rotation() - ? webrtc::kVideoRotation_0 - : static_cast(rotation), - translated_camera_time_us), - width, height); + capturer_->OnFrame( + cricket::WebRtcVideoFrame( + buffer, capturer_->apply_rotation() + ? webrtc::kVideoRotation_0 + : static_cast(rotation), + translated_camera_time_us, 0), + width, height); } void AndroidVideoCapturerJni::OnTextureFrame(int width, @@ -289,16 +289,15 @@ void AndroidVideoCapturerJni::OnTextureFrame(int width, matrix.Rotate(static_cast(rotation)); } - capturer_->OnFrame( - cricket::WebRtcVideoFrame( - surface_texture_helper_->CreateTextureFrame( - adapted_width, adapted_height, - NativeHandleImpl(handle.oes_texture_id, matrix)), - capturer_->apply_rotation() - ? webrtc::kVideoRotation_0 - : static_cast(rotation), - translated_camera_time_us), - width, height); + capturer_->OnFrame(cricket::WebRtcVideoFrame( + surface_texture_helper_->CreateTextureFrame( + adapted_width, adapted_height, + NativeHandleImpl(handle.oes_texture_id, matrix)), + capturer_->apply_rotation() + ? webrtc::kVideoRotation_0 + : static_cast(rotation), + translated_camera_time_us, 0), + width, height); } void AndroidVideoCapturerJni::OnOutputFormatRequest(int width, diff --git a/webrtc/api/androidvideotracksource.cc b/webrtc/api/androidvideotracksource.cc index c52e4f84ff..80e7894a4b 100644 --- a/webrtc/api/androidvideotracksource.cc +++ b/webrtc/api/androidvideotracksource.cc @@ -146,7 +146,7 @@ void AndroidVideoTrackSource::OnByteBufferFrameCaptured(const void* frame_data, buffer, apply_rotation_ ? webrtc::kVideoRotation_0 : static_cast(rotation), - translated_camera_time_us), + translated_camera_time_us, 0), width, height); } @@ -197,7 +197,7 @@ void AndroidVideoTrackSource::OnTextureFrameCaptured( webrtc_jni::NativeHandleImpl(handle.oes_texture_id, matrix)), apply_rotation_ ? webrtc::kVideoRotation_0 : static_cast(rotation), - translated_camera_time_us), + translated_camera_time_us, 0), width, height); } diff --git a/webrtc/media/base/videobroadcaster.cc b/webrtc/media/base/videobroadcaster.cc index ef5c49b4a1..ffe9069ea5 100644 --- a/webrtc/media/base/videobroadcaster.cc +++ b/webrtc/media/base/videobroadcaster.cc @@ -55,8 +55,8 @@ void VideoBroadcaster::OnFrame(const cricket::VideoFrame& frame) { for (auto& sink_pair : sink_pairs()) { if (sink_pair.wants.black_frames) { sink_pair.sink->OnFrame(cricket::WebRtcVideoFrame( - GetBlackFrameBuffer(frame.width(), frame.height()), - frame.rotation(), frame.timestamp_us())); + GetBlackFrameBuffer(frame.width(), frame.height()), frame.rotation(), + frame.timestamp_us(), frame.transport_frame_id())); } else { sink_pair.sink->OnFrame(frame); } diff --git a/webrtc/media/base/videobroadcaster_unittest.cc b/webrtc/media/base/videobroadcaster_unittest.cc index d2948bb437..58efbf3545 100644 --- a/webrtc/media/base/videobroadcaster_unittest.cc +++ b/webrtc/media/base/videobroadcaster_unittest.cc @@ -139,8 +139,8 @@ TEST(VideoBroadcasterTest, SinkWantsBlackFrames) { // Makes it not all black. buffer->InitializeData(); - cricket::WebRtcVideoFrame frame1( - buffer, webrtc::kVideoRotation_0, 10 /* timestamp_us */); + cricket::WebRtcVideoFrame frame1(buffer, webrtc::kVideoRotation_0, + 10 /* timestamp_us */, 0 /* frame_id */); broadcaster.OnFrame(frame1); EXPECT_TRUE(sink1.black_frame()); EXPECT_EQ(10000, sink1.timestamp()); @@ -153,8 +153,8 @@ TEST(VideoBroadcasterTest, SinkWantsBlackFrames) { wants2.black_frames = true; broadcaster.AddOrUpdateSink(&sink2, wants2); - cricket::WebRtcVideoFrame frame2( - buffer, webrtc::kVideoRotation_0, 30 /* timestamp_us */); + cricket::WebRtcVideoFrame frame2(buffer, webrtc::kVideoRotation_0, + 30 /* timestamp_us */, 0 /* frame_id */); broadcaster.OnFrame(frame2); EXPECT_FALSE(sink1.black_frame()); EXPECT_EQ(30000, sink1.timestamp()); diff --git a/webrtc/media/base/videoframe.h b/webrtc/media/base/videoframe.h index 3d765ad5df..a1393499f9 100644 --- a/webrtc/media/base/videoframe.h +++ b/webrtc/media/base/videoframe.h @@ -38,6 +38,9 @@ class VideoFrame { virtual const rtc::scoped_refptr& video_frame_buffer() const = 0; + // Frame ID. Normally RTP timestamp when the frame was received using RTP. + virtual uint32_t transport_frame_id() const = 0; + // System monotonic clock, same timebase as rtc::TimeMicros(). virtual int64_t timestamp_us() const = 0; virtual void set_timestamp_us(int64_t time_us) = 0; diff --git a/webrtc/media/base/videoframefactory.cc b/webrtc/media/base/videoframefactory.cc index 7b89ad6315..a4eeece148 100644 --- a/webrtc/media/base/videoframefactory.cc +++ b/webrtc/media/base/videoframefactory.cc @@ -44,7 +44,8 @@ VideoFrame* VideoFrameFactory::CreateAliasedFrame( scaled_buffer->CropAndScaleFrom(cropped_input_frame->video_frame_buffer()); return new WebRtcVideoFrame(scaled_buffer, cropped_input_frame->rotation(), - cropped_input_frame->timestamp_us()); + cropped_input_frame->timestamp_us(), + cropped_input_frame->transport_frame_id()); } } // namespace cricket diff --git a/webrtc/media/engine/webrtcvideoengine2.cc b/webrtc/media/engine/webrtcvideoengine2.cc index 5da8e33e1e..ef88acd3ae 100644 --- a/webrtc/media/engine/webrtcvideoengine2.cc +++ b/webrtc/media/engine/webrtcvideoengine2.cc @@ -2469,9 +2469,9 @@ void WebRtcVideoChannel2::WebRtcVideoReceiveStream::OnFrame( last_width_ = frame.width(); last_height_ = frame.height(); - const WebRtcVideoFrame render_frame( + WebRtcVideoFrame render_frame( frame.video_frame_buffer(), frame.rotation(), - frame.render_time_ms() * rtc::kNumNanosecsPerMicrosec); + frame.render_time_ms() * rtc::kNumNanosecsPerMicrosec, frame.timestamp()); sink_->OnFrame(render_frame); } diff --git a/webrtc/media/engine/webrtcvideoframe.cc b/webrtc/media/engine/webrtcvideoframe.cc index 2ff4042c52..676f715596 100644 --- a/webrtc/media/engine/webrtcvideoframe.cc +++ b/webrtc/media/engine/webrtcvideoframe.cc @@ -28,9 +28,11 @@ WebRtcVideoFrame::WebRtcVideoFrame() WebRtcVideoFrame::WebRtcVideoFrame( const rtc::scoped_refptr& buffer, webrtc::VideoRotation rotation, - int64_t timestamp_us) + int64_t timestamp_us, + uint32_t transport_frame_id) : video_frame_buffer_(buffer), timestamp_us_(timestamp_us), + transport_frame_id_(transport_frame_id), rotation_(rotation) {} WebRtcVideoFrame::WebRtcVideoFrame( @@ -39,7 +41,8 @@ WebRtcVideoFrame::WebRtcVideoFrame( webrtc::VideoRotation rotation) : WebRtcVideoFrame(buffer, rotation, - time_stamp_ns / rtc::kNumNanosecsPerMicrosec) {} + time_stamp_ns / rtc::kNumNanosecsPerMicrosec, + 0) {} WebRtcVideoFrame::~WebRtcVideoFrame() {} @@ -78,8 +81,25 @@ WebRtcVideoFrame::video_frame_buffer() const { return video_frame_buffer_; } +uint32_t WebRtcVideoFrame::transport_frame_id() const { + return transport_frame_id_; +} + +int64_t WebRtcVideoFrame::timestamp_us() const { + return timestamp_us_; +} + +void WebRtcVideoFrame::set_timestamp_us(int64_t time_us) { + timestamp_us_ = time_us; +} + +webrtc::VideoRotation WebRtcVideoFrame::rotation() const { + return rotation_; +} + VideoFrame* WebRtcVideoFrame::Copy() const { - return new WebRtcVideoFrame(video_frame_buffer_, rotation_, timestamp_us_); + return new WebRtcVideoFrame(video_frame_buffer_, rotation_, timestamp_us_, + transport_frame_id_); } size_t WebRtcVideoFrame::ConvertToRgbBuffer(uint32_t to_fourcc, @@ -195,8 +215,8 @@ const VideoFrame* WebRtcVideoFrame::GetCopyWithRotationApplied() const { current_width, current_height, static_cast(rotation())); if (ret == 0) { - rotated_frame_.reset( - new WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0, timestamp_us_)); + rotated_frame_.reset(new WebRtcVideoFrame( + buffer, webrtc::kVideoRotation_0, timestamp_us_, transport_frame_id_)); } return rotated_frame_.get(); diff --git a/webrtc/media/engine/webrtcvideoframe.h b/webrtc/media/engine/webrtcvideoframe.h index 91fd7b9a90..d7acc6fc14 100644 --- a/webrtc/media/engine/webrtcvideoframe.h +++ b/webrtc/media/engine/webrtcvideoframe.h @@ -38,12 +38,13 @@ class WebRtcVideoFrame : public VideoFrame { // Preferred constructor. WebRtcVideoFrame(const rtc::scoped_refptr& buffer, webrtc::VideoRotation rotation, - int64_t timestamp_us); + int64_t timestamp_us, + uint32_t transport_frame_id); // TODO(nisse): Deprecated, delete as soon as all callers have switched to the // above constructor with microsecond timestamp. WebRtcVideoFrame(const rtc::scoped_refptr& buffer, - int64_t time_stamp_ns, + int64_t timestamp_ns, webrtc::VideoRotation rotation); ~WebRtcVideoFrame(); @@ -59,7 +60,7 @@ class WebRtcVideoFrame : public VideoFrame { int dh, uint8_t* sample, size_t sample_size, - int64_t time_stamp_ns, + int64_t timestamp_ns, webrtc::VideoRotation rotation); // TODO(nisse): We're moving to have all timestamps use the same @@ -79,11 +80,12 @@ class WebRtcVideoFrame : public VideoFrame { const rtc::scoped_refptr& video_frame_buffer() const override; - /* System monotonic clock */ - int64_t timestamp_us() const override { return timestamp_us_; } - void set_timestamp_us(int64_t time_us) override { timestamp_us_ = time_us; }; + uint32_t transport_frame_id() const override; - webrtc::VideoRotation rotation() const override { return rotation_; } + int64_t timestamp_us() const override; + void set_timestamp_us(int64_t time_us) override; + + webrtc::VideoRotation rotation() const override; VideoFrame* Copy() const override; @@ -118,6 +120,7 @@ class WebRtcVideoFrame : public VideoFrame { // An opaque reference counted handle that stores the pixel data. rtc::scoped_refptr video_frame_buffer_; int64_t timestamp_us_; + uint32_t transport_frame_id_; webrtc::VideoRotation rotation_; // This is mutable as the calculation is expensive but once calculated, it diff --git a/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm b/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm index d810095be6..a130d0e7bf 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCVideoRendererAdapter.mm @@ -36,9 +36,9 @@ class VideoRendererAdapter rtc::scoped_refptr i420Buffer = nativeVideoFrame.video_frame_buffer()->NativeToI420Buffer(); std::unique_ptr cpuFrame( - new cricket::WebRtcVideoFrame(i420Buffer, - nativeVideoFrame.rotation(), - nativeVideoFrame.timestamp_us())); + new cricket::WebRtcVideoFrame(i420Buffer, nativeVideoFrame.rotation(), + nativeVideoFrame.timestamp_us(), + nativeVideoFrame.transport_frame_id())); const cricket::VideoFrame *rotatedFrame = cpuFrame->GetCopyWithRotationApplied(); videoFrame = [[RTCVideoFrame alloc] initWithNativeFrame:rotatedFrame]; diff --git a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm index 529499b620..51003cad41 100644 --- a/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm +++ b/webrtc/sdk/objc/Framework/Classes/avfoundationvideocapturer.mm @@ -718,7 +718,7 @@ void AVFoundationVideoCapturer::OnFrameMessage(CVImageBufferRef image_buffer, } OnFrame(cricket::WebRtcVideoFrame(buffer, webrtc::kVideoRotation_0, - translated_camera_time_us), + translated_camera_time_us, 0), captured_width, captured_height); CVBufferRelease(image_buffer);