admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transform encoded frames in RtpVideoStreamReceiver.

Browse Source 
This change is part of the implementation of the Insertable Streams Web
API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
Change-Id: If4ffcfe5761492a2ae5513ec46deb9f837e8aee8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169130
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30755}
This commit is contained in:
Marina Ciocea 2020-03-10 21:31:52 +01:00 committed by Commit Bot
parent 62057627ef
commit 78964c1e0a
7 changed files with 411 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
video/BUILD.gn
View File

@ -28,6 +28,8 @@ rtc_library("video") {
"rtp_streams_synchronizer.h",
"rtp_video_stream_receiver.cc",
"rtp_video_stream_receiver.h",
"rtp_video_stream_receiver_frame_transformer_delegate.cc",
"rtp_video_stream_receiver_frame_transformer_delegate.h",
"send_delay_stats.cc",
"send_delay_stats.h",
"send_statistics_proxy.cc",
@ -99,6 +101,7 @@ rtc_library("video") {
"../modules/video_processing",
"../rtc_base:checks",
"../rtc_base:rate_limiter",
"../rtc_base:rtc_base",
"../rtc_base:rtc_base_approved",
"../rtc_base:rtc_numerics",
"../rtc_base:rtc_task_queue",
@ -503,6 +506,7 @@ if (rtc_include_tests) {
"quality_threshold_unittest.cc",
"receive_statistics_proxy_unittest.cc",
"report_block_stats_unittest.cc",
"rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc",
"rtp_video_stream_receiver_unittest.cc",
"send_delay_stats_unittest.cc",
"send_statistics_proxy_unittest.cc",
@ -532,6 +536,7 @@ if (rtc_include_tests) {
"../api:rtp_parameters",
"../api:scoped_refptr",
"../api:simulated_network_api",
"../api:transport_api",
"../api/crypto:options",
"../api/rtc_event_log",
"../api/task_queue",

27
video/rtp_video_stream_receiver.cc
View File

@ -224,8 +224,7 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver(
packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()),
has_received_frame_(false),
frames_decryptable_(false),
absolute_capture_time_receiver_(clock),
frame_transformer_(frame_transformer) {
absolute_capture_time_receiver_(clock) {
constexpr bool remb_candidate = true;
if (packet_router_)
packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
@ -285,6 +284,13 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver(
buffered_frame_decryptor_->SetFrameDecryptor(std::move(frame_decryptor));
}
}
if (frame_transformer) {
frame_transformer_delegate_ = new rtc::RefCountedObject<
RtpVideoStreamReceiverFrameTransformerDelegate>(
this, std::move(frame_transformer), rtc::Thread::Current());
frame_transformer_delegate_->Init();
}
}
RtpVideoStreamReceiver::RtpVideoStreamReceiver(
@ -326,6 +332,8 @@ RtpVideoStreamReceiver::~RtpVideoStreamReceiver() {
if (packet_router_)
packet_router_->RemoveReceiveRtpModule(rtp_rtcp_.get());
UpdateHistograms();
if (frame_transformer_delegate_)
frame_transformer_delegate_->Reset();
}
void RtpVideoStreamReceiver::AddReceiveCodec(
@ -796,10 +804,13 @@ void RtpVideoStreamReceiver::OnAssembledFrame(
last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
}
if (buffered_frame_decryptor_ == nullptr) {
reference_finder_->ManageFrame(std::move(frame));
} else {
if (buffered_frame_decryptor_ != nullptr) {
buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
} else if (frame_transformer_delegate_) {
frame_transformer_delegate_->TransformFrame(std::move(frame),
config_.rtp.remote_ssrc);
} else {
reference_finder_->ManageFrame(std::move(frame));
}
}
@ -874,6 +885,12 @@ void RtpVideoStreamReceiver::RemoveSecondarySink(
secondary_sinks_.erase(it);
}
void RtpVideoStreamReceiver::ManageFrame(
std::unique_ptr<video_coding::RtpFrameObject> frame) {
rtc::CritScope lock(&reference_finder_lock_);
reference_finder_->ManageFrame(std::move(frame));
}
void RtpVideoStreamReceiver::ReceivePacket(const RtpPacketReceived& packet) {
if (packet.payload_size() == 0) {
// Padding or keep-alive packet.

6
video/rtp_video_stream_receiver.h
View File

@ -48,6 +48,7 @@
#include "rtc_base/thread_annotations.h"
#include "rtc_base/thread_checker.h"
#include "video/buffered_frame_decryptor.h"
#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
namespace webrtc {
@ -198,6 +199,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender,
void AddSecondarySink(RtpPacketSinkInterface* sink);
void RemoveSecondarySink(const RtpPacketSinkInterface* sink);
virtual void ManageFrame(std::unique_ptr<video_coding::RtpFrameObject> frame);
private:
// Used for buffering RTCP feedback messages and sending them all together.
// Note:
@ -370,7 +373,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender,
int64_t last_completed_picture_id_ = 0;
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>
frame_transformer_delegate_;
};
} // namespace webrtc

75
video/rtp_video_stream_receiver_frame_transformer_delegate.cc Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
#include <utility>
#include "absl/memory/memory.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "rtc_base/thread.h"
#include "video/rtp_video_stream_receiver.h"
namespace webrtc {
RtpVideoStreamReceiverFrameTransformerDelegate::
RtpVideoStreamReceiverFrameTransformerDelegate(
RtpVideoStreamReceiver* receiver,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
rtc::Thread* network_thread)
: receiver_(receiver),
frame_transformer_(std::move(frame_transformer)),
network_thread_(network_thread) {}
void RtpVideoStreamReceiverFrameTransformerDelegate::Init() {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
frame_transformer_->RegisterTransformedFrameCallback(
rtc::scoped_refptr<TransformedFrameCallback>(this));
}
void RtpVideoStreamReceiverFrameTransformerDelegate::Reset() {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
frame_transformer_->UnregisterTransformedFrameCallback();
frame_transformer_ = nullptr;
receiver_ = nullptr;
}
void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame(
std::unique_ptr<video_coding::RtpFrameObject> frame,
uint32_t ssrc) {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
auto additional_data =
RtpDescriptorAuthentication(frame->GetRtpVideoHeader());
frame_transformer_->TransformFrame(std::move(frame),
std::move(additional_data), ssrc);
}
void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
std::unique_ptr<video_coding::EncodedFrame> frame) {
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate =
this;
network_thread_->PostTask(ToQueuedTask(
[delegate = std::move(delegate), frame = std::move(frame)]() mutable {
delegate->ManageFrame(std::move(frame));
}));
}
void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
std::unique_ptr<video_coding::EncodedFrame> frame) {
RTC_DCHECK_RUN_ON(&network_sequence_checker_);
if (!receiver_)
return;
auto transformed_frame = absl::WrapUnique(
static_cast<video_coding::RtpFrameObject*>(frame.release()));
receiver_->ManageFrame(std::move(transformed_frame));
}
} // namespace webrtc

64
video/rtp_video_stream_receiver_frame_transformer_delegate.h Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef VIDEO_RTP_VIDEO_STREAM_RECEIVER_FRAME_TRANSFORMER_DELEGATE_H_
#define VIDEO_RTP_VIDEO_STREAM_RECEIVER_FRAME_TRANSFORMER_DELEGATE_H_
#include <memory>
#include "api/frame_transformer_interface.h"
#include "modules/video_coding/frame_object.h"
#include "rtc_base/synchronization/sequence_checker.h"
#include "rtc_base/thread.h"
namespace webrtc {
class RtpVideoStreamReceiver;
// Delegates calls to FrameTransformerInterface to transform frames, and to
// RtpVideoStreamReceiver to manage transformed frames on the |network_thread_|.
class RtpVideoStreamReceiverFrameTransformerDelegate
: public TransformedFrameCallback {
public:
RtpVideoStreamReceiverFrameTransformerDelegate(
RtpVideoStreamReceiver* receiver,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
rtc::Thread* network_thread);
void Init();
void Reset();
// Delegates the call to FrameTransformerInterface::TransformFrame.
void TransformFrame(std::unique_ptr<video_coding::RtpFrameObject> frame,
uint32_t ssrc);
// Implements TransformedFrameCallback. Can be called on any thread. Posts
// the transformed frame to be managed on the |network_thread_|.
void OnTransformedFrame(
std::unique_ptr<video_coding::EncodedFrame> frame) override;
// Delegates the call to RtpVideoReceiver::ManageFrame on the
// |network_thread_|.
void ManageFrame(std::unique_ptr<video_coding::EncodedFrame> frame);
protected:
~RtpVideoStreamReceiverFrameTransformerDelegate() override = default;
private:
SequenceChecker network_sequence_checker_;
RtpVideoStreamReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_);
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
RTC_GUARDED_BY(network_sequence_checker_);
rtc::Thread* const network_thread_;
};
} // namespace webrtc
#endif // VIDEO_RTP_VIDEO_STREAM_RECEIVER_FRAME_TRANSFORMER_DELEGATE_H_

199
video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc Normal file
View File

@ -0,0 +1,199 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
#include <cstdio>
#include <memory>
#include <utility>
#include <vector>
#include "api/call/transport.h"
#include "call/video_receive_stream.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "modules/utility/include/process_thread.h"
#include "rtc_base/event.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "video/rtp_video_stream_receiver.h"
namespace webrtc {
namespace {
using ::testing::_;
std::unique_ptr<video_coding::RtpFrameObject> CreateRtpFrameObject() {
return std::make_unique<video_coding::RtpFrameObject>(
0, 0, true, 0, 0, 0, 0, 0, VideoSendTiming(), 0, kVideoCodecGeneric,
kVideoRotation_0, VideoContentType::UNSPECIFIED, RTPVideoHeader(),
absl::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(0));
}
class FakeTransport : public Transport {
public:
bool SendRtp(const uint8_t* packet,
size_t length,
const PacketOptions& options) {
return true;
}
bool SendRtcp(const uint8_t* packet, size_t length) { return true; }
};
class FakeNackSender : public NackSender {
public:
void SendNack(const std::vector<uint16_t>& sequence_numbers) {}
void SendNack(const std::vector<uint16_t>& sequence_numbers,
bool buffering_allowed) {}
};
class FakeOnCompleteFrameCallback
: public video_coding::OnCompleteFrameCallback {
public:
void OnCompleteFrame(
std::unique_ptr<video_coding::EncodedFrame> frame) override {}
};
class TestRtpVideoStreamReceiverInitializer {
public:
TestRtpVideoStreamReceiverInitializer()
: test_config_(nullptr),
test_process_thread_(ProcessThread::Create("TestThread")) {
test_config_.rtp.remote_ssrc = 1111;
test_config_.rtp.local_ssrc = 2222;
test_rtp_receive_statistics_ =
ReceiveStatistics::Create(Clock::GetRealTimeClock());
}
protected:
VideoReceiveStream::Config test_config_;
FakeTransport fake_transport_;
FakeNackSender fake_nack_sender_;
FakeOnCompleteFrameCallback fake_on_complete_frame_callback_;
std::unique_ptr<ProcessThread> test_process_thread_;
std::unique_ptr<ReceiveStatistics> test_rtp_receive_statistics_;
};
class TestRtpVideoStreamReceiver : public TestRtpVideoStreamReceiverInitializer,
public RtpVideoStreamReceiver {
public:
TestRtpVideoStreamReceiver()
: TestRtpVideoStreamReceiverInitializer(),
RtpVideoStreamReceiver(Clock::GetRealTimeClock(),
&fake_transport_,
nullptr,
nullptr,
&test_config_,
test_rtp_receive_statistics_.get(),
nullptr,
test_process_thread_.get(),
&fake_nack_sender_,
nullptr,
&fake_on_complete_frame_callback_,
nullptr,
nullptr) {}
~TestRtpVideoStreamReceiver() override = default;
MOCK_METHOD(void,
ManageFrame,
(std::unique_ptr<video_coding::RtpFrameObject> frame),
(override));
};
class MockFrameTransformer : public FrameTransformerInterface {
public:
~MockFrameTransformer() override = default;
MOCK_METHOD(void,
TransformFrame,
(std::unique_ptr<video_coding::EncodedFrame>,
std::vector<uint8_t>,
uint32_t),
(override));
MOCK_METHOD(void,
RegisterTransformedFrameCallback,
(rtc::scoped_refptr<TransformedFrameCallback>),
(override));
MOCK_METHOD(void, UnregisterTransformedFrameCallback, (), (override));
};
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
RegisterTransformedFrameCallbackOnInit) {
TestRtpVideoStreamReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<MockFrameTransformer>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
new rtc::RefCountedObject<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, rtc::Thread::Current()));
EXPECT_CALL(*frame_transformer, RegisterTransformedFrameCallback);
delegate->Init();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
UnregisterTransformedFrameCallbackOnReset) {
TestRtpVideoStreamReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<MockFrameTransformer>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
new rtc::RefCountedObject<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, rtc::Thread::Current()));
EXPECT_CALL(*frame_transformer, UnregisterTransformedFrameCallback);
delegate->Reset();
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, TransformFrame) {
TestRtpVideoStreamReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<MockFrameTransformer>());
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate(
new rtc::RefCountedObject<RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, rtc::Thread::Current()));
auto frame = CreateRtpFrameObject();
EXPECT_CALL(*frame_transformer,
TransformFrame(_, RtpDescriptorAuthentication(RTPVideoHeader()),
/*remote_ssrc*/ 1111));
delegate->TransformFrame(std::move(frame), /*remote_ssrc*/ 1111);
}
TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest,
ManageFrameOnTransformedFrame) {
auto main_thread = rtc::Thread::Create();
main_thread->Start();
auto network_thread = rtc::Thread::Create();
network_thread->Start();
TestRtpVideoStreamReceiver receiver;
rtc::scoped_refptr<MockFrameTransformer> frame_transformer(
new rtc::RefCountedObject<MockFrameTransformer>());
auto delegate = network_thread->Invoke<
rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate>>(
RTC_FROM_HERE, [&]() mutable {
return new rtc::RefCountedObject<
RtpVideoStreamReceiverFrameTransformerDelegate>(
&receiver, frame_transformer, network_thread.get());
});
auto frame = CreateRtpFrameObject();
EXPECT_CALL(receiver, ManageFrame)
.WillOnce([&network_thread](
std::unique_ptr<video_coding::RtpFrameObject> frame) {
EXPECT_TRUE(network_thread->IsCurrent());
});
main_thread->Invoke<void>(RTC_FROM_HERE, [&]() mutable {
delegate->OnTransformedFrame(std::move(frame));
});
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
main_thread->Stop();
network_thread->Stop();
}
} // namespace
} // namespace webrtc

41
video/rtp_video_stream_receiver_unittest.cc
View File

@ -17,6 +17,7 @@
#include "api/video/video_frame_type.h"
#include "common_video/h264/h264_common.h"
#include "media/base/media_constants.h"
#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
#include "modules/rtp_rtcp/source/rtp_format.h"
#include "modules/rtp_rtcp/source/rtp_format_vp9.h"
#include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h"
@ -124,6 +125,17 @@ class MockRtpPacketSink : public RtpPacketSinkInterface {
MOCK_METHOD1(OnRtpPacket, void(const RtpPacketReceived&));
};
class MockFrameTransformer : public FrameTransformerInterface {
public:
MOCK_METHOD3(TransformFrame,
void(std::unique_ptr<video_coding::EncodedFrame> frame,
std::vector<uint8_t> additional_data,
uint32_t ssrc));
MOCK_METHOD1(RegisterTransformedFrameCallback,
void(rtc::scoped_refptr<TransformedFrameCallback>));
MOCK_METHOD0(UnregisterTransformedFrameCallback, void());
};
constexpr uint32_t kSsrc = 111;
constexpr uint16_t kSequenceNumber = 222;
std::unique_ptr<RtpPacketReceived> CreateRtpPacketReceived(
@ -1205,4 +1217,33 @@ TEST_F(RtpVideoStreamReceiverTest, RepeatedSecondarySinkDisallowed) {
}
#endif
TEST_F(RtpVideoStreamReceiverTest, TransformFrame) {
rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer =
new rtc::RefCountedObject<MockFrameTransformer>();
EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameCallback);
auto receiver = std::make_unique<RtpVideoStreamReceiver>(
Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_,
rtp_receive_statistics_.get(), nullptr, process_thread_.get(),
&mock_nack_sender_, nullptr, &mock_on_complete_frame_callback_, nullptr,
mock_frame_transformer);
RtpPacketReceived rtp_packet;
RTPVideoHeader video_header;
rtc::CopyOnWriteBuffer data({1, 2, 3, 4});
rtp_packet.SetSequenceNumber(1);
video_header.is_first_packet_in_frame = true;
video_header.is_last_packet_in_frame = true;
video_header.codec = kVideoCodecGeneric;
video_header.frame_type = VideoFrameType::kVideoFrameKey;
mock_on_complete_frame_callback_.AppendExpectedBitstream(data.data(),
data.size());
EXPECT_CALL(*mock_frame_transformer,
TransformFrame(_, RtpDescriptorAuthentication(video_header),
config_.rtp.remote_ssrc));
receiver->OnReceivedPayloadData(data, rtp_packet, video_header);
EXPECT_CALL(*mock_frame_transformer, UnregisterTransformedFrameCallback());
receiver = nullptr;
}
} // namespace webrtc