Allow setting different number of temporal layers per simulcast layer.
Setting different number of temporal layers is supported by SimulcastEncodeAdapter and LibvpxVp8Encoder will fallback to SimulcastEncoderAdapter if InitEncode fails. Bug: none Change-Id: I8a09ee1e6c70a0006317957c0802d019a0d28ca2 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228642 Reviewed-by: Erik Språng <sprang@webrtc.org> Reviewed-by: Harald Alvestrand <hta@webrtc.org> Commit-Queue: Åsa Persson <asapersson@webrtc.org> Cr-Commit-Position: refs/heads/master@{#34785}
This commit is contained in:
Åsa Persson
WebRTC LUCI CQ
parent
29dddff209
commit
fb1959625d
@ -489,8 +489,6 @@ struct RTC_EXPORT RtpEncodingParameters {
|
||||
|
||||
// Specifies the number of temporal layers for video (if the feature is
|
||||
// supported by the codec implementation).
|
||||
// TODO(asapersson): Different number of temporal layers are not supported
|
||||
// per simulcast layer.
|
||||
// Screencast support is experimental.
|
||||
absl::optional<int> num_temporal_layers;
|
||||
|
||||
|
||||
@ -106,15 +106,6 @@ webrtc::RTCError CheckRtpParametersValues(
|
||||
"num_temporal_layers to an invalid number.");
|
||||
}
|
||||
}
|
||||
if (i > 0 && (rtp_parameters.encodings[i].num_temporal_layers !=
|
||||
rtp_parameters.encodings[i - 1].num_temporal_layers)) {
|
||||
LOG_AND_RETURN_ERROR(
|
||||
RTCErrorType::INVALID_MODIFICATION,
|
||||
"Attempted to set RtpParameters num_temporal_layers "
|
||||
"at encoding layer i: " +
|
||||
rtc::ToString(i) +
|
||||
" to a different value than other encoding layers.");
|
||||
}
|
||||
}
|
||||
|
||||
return webrtc::RTCError::OK();
|
||||
|
||||
@ -7706,28 +7706,6 @@ TEST_F(WebRtcVideoChannelTest,
|
||||
channel_->SetRtpSendParameters(last_ssrc_, parameters).type());
|
||||
}
|
||||
|
||||
TEST_F(WebRtcVideoChannelTest,
|
||||
SetRtpSendParametersNumTemporalLayersFailsForInvalidModification) {
|
||||
const size_t kNumSimulcastStreams = 3;
|
||||
SetUpSimulcast(true, false);
|
||||
|
||||
// Get and set the rtp encoding parameters.
|
||||
webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_);
|
||||
EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
|
||||
|
||||
// No/all layers should be set.
|
||||
parameters.encodings[0].num_temporal_layers = 1;
|
||||
EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION,
|
||||
channel_->SetRtpSendParameters(last_ssrc_, parameters).type());
|
||||
|
||||
// Different values not supported.
|
||||
parameters.encodings[0].num_temporal_layers = 1;
|
||||
parameters.encodings[1].num_temporal_layers = 2;
|
||||
parameters.encodings[2].num_temporal_layers = 2;
|
||||
EXPECT_EQ(webrtc::RTCErrorType::INVALID_MODIFICATION,
|
||||
channel_->SetRtpSendParameters(last_ssrc_, parameters).type());
|
||||
}
|
||||
|
||||
TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersNumTemporalLayers) {
|
||||
const size_t kNumSimulcastStreams = 3;
|
||||
SetUpSimulcast(true, false);
|
||||
@ -7767,9 +7745,9 @@ TEST_F(WebRtcVideoChannelTest, NumTemporalLayersPropagatedToEncoder) {
|
||||
// Change the value and set it on the VideoChannel.
|
||||
webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_);
|
||||
EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
|
||||
parameters.encodings[0].num_temporal_layers = 2;
|
||||
parameters.encodings[0].num_temporal_layers = 3;
|
||||
parameters.encodings[1].num_temporal_layers = 2;
|
||||
parameters.encodings[2].num_temporal_layers = 2;
|
||||
parameters.encodings[2].num_temporal_layers = 1;
|
||||
EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok());
|
||||
|
||||
// Verify that the new value is propagated down to the encoder.
|
||||
@ -7778,16 +7756,16 @@ TEST_F(WebRtcVideoChannelTest, NumTemporalLayersPropagatedToEncoder) {
|
||||
webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy();
|
||||
EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams);
|
||||
EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size());
|
||||
EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers);
|
||||
EXPECT_EQ(3UL, encoder_config.simulcast_layers[0].num_temporal_layers);
|
||||
EXPECT_EQ(2UL, encoder_config.simulcast_layers[1].num_temporal_layers);
|
||||
EXPECT_EQ(2UL, encoder_config.simulcast_layers[2].num_temporal_layers);
|
||||
EXPECT_EQ(1UL, encoder_config.simulcast_layers[2].num_temporal_layers);
|
||||
|
||||
// FakeVideoSendStream calls CreateEncoderStreams, test that the vector of
|
||||
// VideoStreams are created appropriately for the simulcast case.
|
||||
EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size());
|
||||
EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers);
|
||||
EXPECT_EQ(3UL, stream->GetVideoStreams()[0].num_temporal_layers);
|
||||
EXPECT_EQ(2UL, stream->GetVideoStreams()[1].num_temporal_layers);
|
||||
EXPECT_EQ(2UL, stream->GetVideoStreams()[2].num_temporal_layers);
|
||||
EXPECT_EQ(1UL, stream->GetVideoStreams()[2].num_temporal_layers);
|
||||
|
||||
// No parameter changed, encoder should not be reconfigured.
|
||||
EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok());
|
||||
@ -7809,29 +7787,28 @@ TEST_F(WebRtcVideoChannelTest,
|
||||
channel_->SetSend(true);
|
||||
frame_forwarder.IncomingCapturedFrame(frame_source_.GetFrame());
|
||||
|
||||
// Change rtp encoding parameters, num_temporal_layers not changed.
|
||||
// Change rtp encoding parameters.
|
||||
webrtc::RtpParameters parameters = channel_->GetRtpSendParameters(last_ssrc_);
|
||||
EXPECT_EQ(kNumSimulcastStreams, parameters.encodings.size());
|
||||
parameters.encodings[0].min_bitrate_bps = 33000;
|
||||
parameters.encodings[0].num_temporal_layers = 2;
|
||||
parameters.encodings[2].num_temporal_layers = 1;
|
||||
EXPECT_TRUE(channel_->SetRtpSendParameters(last_ssrc_, parameters).ok());
|
||||
|
||||
// Verify that no value is propagated down to the encoder.
|
||||
webrtc::VideoEncoderConfig encoder_config = stream->GetEncoderConfig().Copy();
|
||||
EXPECT_EQ(kNumSimulcastStreams, encoder_config.number_of_streams);
|
||||
EXPECT_EQ(kNumSimulcastStreams, encoder_config.simulcast_layers.size());
|
||||
EXPECT_FALSE(encoder_config.simulcast_layers[0].num_temporal_layers);
|
||||
EXPECT_EQ(2UL, encoder_config.simulcast_layers[0].num_temporal_layers);
|
||||
EXPECT_FALSE(encoder_config.simulcast_layers[1].num_temporal_layers);
|
||||
EXPECT_FALSE(encoder_config.simulcast_layers[2].num_temporal_layers);
|
||||
EXPECT_EQ(1UL, encoder_config.simulcast_layers[2].num_temporal_layers);
|
||||
|
||||
// FakeVideoSendStream calls CreateEncoderStreams, test that the vector of
|
||||
// VideoStreams are created appropriately for the simulcast case.
|
||||
EXPECT_EQ(kNumSimulcastStreams, stream->GetVideoStreams().size());
|
||||
EXPECT_EQ(kDefaultNumTemporalLayers,
|
||||
stream->GetVideoStreams()[0].num_temporal_layers);
|
||||
EXPECT_EQ(2UL, stream->GetVideoStreams()[0].num_temporal_layers);
|
||||
EXPECT_EQ(kDefaultNumTemporalLayers,
|
||||
stream->GetVideoStreams()[1].num_temporal_layers);
|
||||
EXPECT_EQ(kDefaultNumTemporalLayers,
|
||||
stream->GetVideoStreams()[2].num_temporal_layers);
|
||||
EXPECT_EQ(1UL, stream->GetVideoStreams()[2].num_temporal_layers);
|
||||
|
||||
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
|
||||
}
|
||||
|
||||
@ -1260,6 +1260,30 @@ TEST_F(RtpSenderReceiverTest, VideoSenderDetectInvalidScaleResolutionDownBy) {
|
||||
DestroyVideoRtpSender();
|
||||
}
|
||||
|
||||
TEST_F(RtpSenderReceiverTest, VideoSenderCanSetNumTemporalLayers) {
|
||||
CreateVideoRtpSender();
|
||||
|
||||
RtpParameters params = video_rtp_sender_->GetParameters();
|
||||
params.encodings[0].num_temporal_layers = 2;
|
||||
|
||||
EXPECT_TRUE(video_rtp_sender_->SetParameters(params).ok());
|
||||
params = video_rtp_sender_->GetParameters();
|
||||
EXPECT_EQ(2, params.encodings[0].num_temporal_layers);
|
||||
|
||||
DestroyVideoRtpSender();
|
||||
}
|
||||
|
||||
TEST_F(RtpSenderReceiverTest, VideoSenderDetectInvalidNumTemporalLayers) {
|
||||
CreateVideoRtpSender();
|
||||
|
||||
RtpParameters params = video_rtp_sender_->GetParameters();
|
||||
params.encodings[0].num_temporal_layers = webrtc::kMaxTemporalStreams + 1;
|
||||
RTCError result = video_rtp_sender_->SetParameters(params);
|
||||
EXPECT_EQ(RTCErrorType::INVALID_RANGE, result.type());
|
||||
|
||||
DestroyVideoRtpSender();
|
||||
}
|
||||
|
||||
TEST_F(RtpSenderReceiverTest, VideoSenderCanSetMaxFramerate) {
|
||||
CreateVideoRtpSender();
|
||||
|
||||
|
||||
@ -25,13 +25,18 @@
|
||||
#include "call/rtp_transport_controller_send.h"
|
||||
#include "call/simulated_network.h"
|
||||
#include "call/video_send_stream.h"
|
||||
#include "media/engine/internal_encoder_factory.h"
|
||||
#include "media/engine/simulcast_encoder_adapter.h"
|
||||
#include "media/engine/webrtc_video_engine.h"
|
||||
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
|
||||
#include "modules/rtp_rtcp/source/create_video_rtp_depacketizer.h"
|
||||
#include "modules/rtp_rtcp/source/rtcp_sender.h"
|
||||
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
|
||||
#include "modules/rtp_rtcp/source/rtp_packet.h"
|
||||
#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
|
||||
#include "modules/rtp_rtcp/source/rtp_util.h"
|
||||
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_vp9.h"
|
||||
#include "modules/video_coding/codecs/interface/common_constants.h"
|
||||
#include "modules/video_coding/codecs/vp8/include/vp8.h"
|
||||
#include "modules/video_coding/codecs/vp9/include/vp9.h"
|
||||
#include "rtc_base/checks.h"
|
||||
@ -122,6 +127,10 @@ class VideoSendStreamTest : public test::CallTest {
|
||||
uint8_t num_spatial_layers);
|
||||
|
||||
void TestRequestSourceRotateVideo(bool support_orientation_ext);
|
||||
|
||||
void TestTemporalLayers(VideoEncoderFactory* encoder_factory,
|
||||
const std::string& payload_name,
|
||||
const std::vector<int>& num_temporal_layers);
|
||||
};
|
||||
|
||||
TEST_F(VideoSendStreamTest, CanStartStartedStream) {
|
||||
@ -3971,4 +3980,204 @@ TEST_F(VideoSendStreamTest, SwitchesToScreenshareAndBack) {
|
||||
RunBaseTest(&test);
|
||||
}
|
||||
|
||||
void VideoSendStreamTest::TestTemporalLayers(
|
||||
VideoEncoderFactory* encoder_factory,
|
||||
const std::string& payload_name,
|
||||
const std::vector<int>& num_temporal_layers) {
|
||||
static constexpr int kMaxBitrateBps = 1000000;
|
||||
static constexpr int kMinFramesToObservePerStream = 8;
|
||||
|
||||
class TemporalLayerObserver
|
||||
: public test::EndToEndTest,
|
||||
public test::FrameGeneratorCapturer::SinkWantsObserver {
|
||||
public:
|
||||
TemporalLayerObserver(VideoEncoderFactory* encoder_factory,
|
||||
const std::string& payload_name,
|
||||
const std::vector<int>& num_temporal_layers)
|
||||
: EndToEndTest(kDefaultTimeoutMs),
|
||||
encoder_factory_(encoder_factory),
|
||||
payload_name_(payload_name),
|
||||
num_temporal_layers_(num_temporal_layers),
|
||||
depacketizer_(CreateVideoRtpDepacketizer(
|
||||
PayloadStringToCodecType(payload_name))) {}
|
||||
|
||||
private:
|
||||
void OnFrameGeneratorCapturerCreated(
|
||||
test::FrameGeneratorCapturer* frame_generator_capturer) override {
|
||||
frame_generator_capturer->ChangeResolution(640, 360);
|
||||
}
|
||||
|
||||
void OnSinkWantsChanged(rtc::VideoSinkInterface<VideoFrame>* sink,
|
||||
const rtc::VideoSinkWants& wants) override {}
|
||||
|
||||
void ModifySenderBitrateConfig(
|
||||
BitrateConstraints* bitrate_config) override {
|
||||
bitrate_config->start_bitrate_bps = kMaxBitrateBps / 2;
|
||||
}
|
||||
|
||||
size_t GetNumVideoStreams() const override {
|
||||
return num_temporal_layers_.size();
|
||||
}
|
||||
|
||||
void ModifyVideoConfigs(
|
||||
VideoSendStream::Config* send_config,
|
||||
std::vector<VideoReceiveStream::Config>* receive_configs,
|
||||
VideoEncoderConfig* encoder_config) override {
|
||||
send_config->encoder_settings.encoder_factory = encoder_factory_;
|
||||
send_config->rtp.payload_name = payload_name_;
|
||||
send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType;
|
||||
encoder_config->video_format.name = payload_name_;
|
||||
encoder_config->codec_type = PayloadStringToCodecType(payload_name_);
|
||||
encoder_config->video_stream_factory =
|
||||
rtc::make_ref_counted<cricket::EncoderStreamFactory>(
|
||||
payload_name_, /*max_qp=*/56, /*is_screenshare=*/false,
|
||||
/*conference_mode=*/false);
|
||||
encoder_config->max_bitrate_bps = kMaxBitrateBps;
|
||||
|
||||
for (size_t i = 0; i < num_temporal_layers_.size(); ++i) {
|
||||
VideoStream& stream = encoder_config->simulcast_layers[i];
|
||||
stream.num_temporal_layers = num_temporal_layers_[i];
|
||||
configured_num_temporal_layers_[send_config->rtp.ssrcs[i]] =
|
||||
num_temporal_layers_[i];
|
||||
}
|
||||
}
|
||||
|
||||
struct ParsedPacket {
|
||||
uint32_t timestamp;
|
||||
uint32_t ssrc;
|
||||
int temporal_idx;
|
||||
};
|
||||
|
||||
bool ParsePayload(const uint8_t* packet,
|
||||
size_t length,
|
||||
ParsedPacket& parsed) const {
|
||||
RtpPacket rtp_packet;
|
||||
EXPECT_TRUE(rtp_packet.Parse(packet, length));
|
||||
|
||||
if (rtp_packet.payload_size() == 0) {
|
||||
return false; // Padding packet.
|
||||
}
|
||||
parsed.timestamp = rtp_packet.Timestamp();
|
||||
parsed.ssrc = rtp_packet.Ssrc();
|
||||
|
||||
absl::optional<VideoRtpDepacketizer::ParsedRtpPayload> parsed_payload =
|
||||
depacketizer_->Parse(rtp_packet.PayloadBuffer());
|
||||
EXPECT_TRUE(parsed_payload);
|
||||
|
||||
if (const auto* vp8_header = absl::get_if<RTPVideoHeaderVP8>(
|
||||
&parsed_payload->video_header.video_type_header)) {
|
||||
parsed.temporal_idx = vp8_header->temporalIdx;
|
||||
} else {
|
||||
RTC_NOTREACHED();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Action OnSendRtp(const uint8_t* packet, size_t length) override {
|
||||
ParsedPacket parsed;
|
||||
if (!ParsePayload(packet, length, parsed))
|
||||
return SEND_PACKET;
|
||||
|
||||
uint32_t ssrc = parsed.ssrc;
|
||||
int temporal_idx =
|
||||
parsed.temporal_idx == kNoTemporalIdx ? 0 : parsed.temporal_idx;
|
||||
max_observed_tl_idxs_[ssrc] =
|
||||
std::max(temporal_idx, max_observed_tl_idxs_[ssrc]);
|
||||
|
||||
if (last_observed_packet_.count(ssrc) == 0 ||
|
||||
parsed.timestamp != last_observed_packet_[ssrc].timestamp) {
|
||||
num_observed_frames_[ssrc]++;
|
||||
}
|
||||
last_observed_packet_[ssrc] = parsed;
|
||||
|
||||
if (HighestTemporalLayerSentPerStream())
|
||||
observation_complete_.Set();
|
||||
|
||||
return SEND_PACKET;
|
||||
}
|
||||
|
||||
bool HighestTemporalLayerSentPerStream() const {
|
||||
if (num_observed_frames_.size() !=
|
||||
configured_num_temporal_layers_.size()) {
|
||||
return false;
|
||||
}
|
||||
for (const auto& num_frames : num_observed_frames_) {
|
||||
if (num_frames.second < kMinFramesToObservePerStream) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (max_observed_tl_idxs_.size() !=
|
||||
configured_num_temporal_layers_.size()) {
|
||||
return false;
|
||||
}
|
||||
for (const auto& max_tl_idx : max_observed_tl_idxs_) {
|
||||
uint32_t ssrc = max_tl_idx.first;
|
||||
int configured_num_tls =
|
||||
configured_num_temporal_layers_.find(ssrc)->second;
|
||||
if (max_tl_idx.second != configured_num_tls - 1)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void PerformTest() override { EXPECT_TRUE(Wait()); }
|
||||
|
||||
VideoEncoderFactory* const encoder_factory_;
|
||||
const std::string payload_name_;
|
||||
const std::vector<int> num_temporal_layers_;
|
||||
const std::unique_ptr<VideoRtpDepacketizer> depacketizer_;
|
||||
// Mapped by SSRC.
|
||||
std::map<uint32_t, int> configured_num_temporal_layers_;
|
||||
std::map<uint32_t, int> max_observed_tl_idxs_;
|
||||
std::map<uint32_t, int> num_observed_frames_;
|
||||
std::map<uint32_t, ParsedPacket> last_observed_packet_;
|
||||
} test(encoder_factory, payload_name, num_temporal_layers);
|
||||
|
||||
RunBaseTest(&test);
|
||||
}
|
||||
|
||||
TEST_F(VideoSendStreamTest, TestTemporalLayersVp8) {
|
||||
InternalEncoderFactory internal_encoder_factory;
|
||||
test::FunctionVideoEncoderFactory encoder_factory(
|
||||
[&internal_encoder_factory]() {
|
||||
return std::make_unique<SimulcastEncoderAdapter>(
|
||||
&internal_encoder_factory, SdpVideoFormat("VP8"));
|
||||
});
|
||||
|
||||
TestTemporalLayers(&encoder_factory, "VP8",
|
||||
/*num_temporal_layers=*/{2});
|
||||
}
|
||||
|
||||
TEST_F(VideoSendStreamTest, TestTemporalLayersVp8Simulcast) {
|
||||
InternalEncoderFactory internal_encoder_factory;
|
||||
test::FunctionVideoEncoderFactory encoder_factory(
|
||||
[&internal_encoder_factory]() {
|
||||
return std::make_unique<SimulcastEncoderAdapter>(
|
||||
&internal_encoder_factory, SdpVideoFormat("VP8"));
|
||||
});
|
||||
|
||||
TestTemporalLayers(&encoder_factory, "VP8",
|
||||
/*num_temporal_layers=*/{2, 2});
|
||||
}
|
||||
|
||||
TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithDifferentNumTls) {
|
||||
InternalEncoderFactory internal_encoder_factory;
|
||||
test::FunctionVideoEncoderFactory encoder_factory(
|
||||
[&internal_encoder_factory]() {
|
||||
return std::make_unique<SimulcastEncoderAdapter>(
|
||||
&internal_encoder_factory, SdpVideoFormat("VP8"));
|
||||
});
|
||||
|
||||
TestTemporalLayers(&encoder_factory, "VP8",
|
||||
/*num_temporal_layers=*/{3, 1});
|
||||
}
|
||||
|
||||
TEST_F(VideoSendStreamTest, TestTemporalLayersVp8SimulcastWithoutSimAdapter) {
|
||||
test::FunctionVideoEncoderFactory encoder_factory(
|
||||
[]() { return VP8Encoder::Create(); });
|
||||
|
||||
TestTemporalLayers(&encoder_factory, "VP8",
|
||||
/*num_temporal_layers=*/{2, 2});
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user