admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reland of Add experimental simulcast screen content mode

Browse Source 
The original CL was reverted because of a bug discovered by the
chromium bots. Description of that CL:

> Review-Url: https://codereview.webrtc.org/2636443002
> Cr-Commit-Position: refs/heads/master@{#16135}
> Committed: a28e971e3b

The first patch set of this CL is the same as r16135.
Subsequence patch sets are the fixes applied.
Some new test cases have been added, which reveal a few more bugs that
have also been fixed.

BUG=webrtc:4172

Review-Url: https://codereview.webrtc.org/2641133002
Cr-Commit-Position: refs/heads/master@{#16299}
This commit is contained in:
sprang 2017-01-26 06:12:26 -08:00 committed by Commit bot
parent 4b15d9a7f1
commit 429600d7d0
12 changed files with 421 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
webrtc/media/engine/simulcast.cc
View File

@ -13,6 +13,7 @@
#include "webrtc/base/arraysize.h"
#include "webrtc/base/logging.h"
#include "webrtc/media/base/streamparams.h"
#include "webrtc/media/engine/constants.h"
#include "webrtc/media/engine/simulcast.h"
#include "webrtc/system_wrappers/include/field_trial.h"
@ -48,6 +49,8 @@ const SimulcastFormat kSimulcastFormats[] = {
{0, 0, 1, 200, 150, 30}
};
const int kDefaultScreenshareSimulcastStreams = 2;
// Multiway: Number of temporal layers for each simulcast stream, for maximum
// possible number of simulcast streams |kMaxSimulcastStreams|. The array
// goes from lowest resolution at position 0 to highest resolution.
@ -78,8 +81,8 @@ int FindSimulcastFormatIndex(int width, int height) {
MaybeExchangeWidthHeight(&width, &height);
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
if (width >= kSimulcastFormats[i].width &&
height >= kSimulcastFormats[i].height) {
if (width * height >=
kSimulcastFormats[i].width * kSimulcastFormats[i].height) {
return i;
}
}
@ -90,8 +93,8 @@ int FindSimulcastFormatIndex(int width, int height, size_t max_layers) {
MaybeExchangeWidthHeight(&width, &height);
for (uint32_t i = 0; i < arraysize(kSimulcastFormats); ++i) {
if (width >= kSimulcastFormats[i].width &&
height >= kSimulcastFormats[i].height &&
if (width * height >=
kSimulcastFormats[i].width * kSimulcastFormats[i].height &&
max_layers == kSimulcastFormats[i].max_layers) {
return i;
}
@ -117,7 +120,7 @@ size_t FindSimulcastMaxLayers(int width, int height) {
// TODO(marpan): Investigate if we should return 0 instead of -1 in
// FindSimulcast[Max/Target/Min]Bitrate functions below, since the
// codec struct max/min/targeBitrates are unsigned.
int FindSimulcastMaxBitrateBps(int width, int height, size_t max_layers) {
int FindSimulcastMaxBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@ -125,9 +128,7 @@ int FindSimulcastMaxBitrateBps(int width, int height, size_t max_layers) {
return kSimulcastFormats[format_index].max_bitrate_kbps * 1000;
}
int FindSimulcastTargetBitrateBps(int width,
int height,
size_t max_layers) {
int FindSimulcastTargetBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@ -135,7 +136,7 @@ int FindSimulcastTargetBitrateBps(int width,
return kSimulcastFormats[format_index].target_bitrate_kbps * 1000;
}
int FindSimulcastMinBitrateBps(int width, int height, size_t max_layers) {
int FindSimulcastMinBitrateBps(int width, int height) {
const int format_index = FindSimulcastFormatIndex(width, height);
if (format_index == -1) {
return -1;
@ -166,52 +167,73 @@ int GetTotalMaxBitrateBps(const std::vector<webrtc::VideoStream>& streams) {
return total_max_bitrate_bps;
}
std::vector<webrtc::VideoStream> GetSimulcastConfig(
size_t max_streams,
int width,
int height,
int max_bitrate_bps,
int max_qp,
int max_framerate) {
size_t simulcast_layers = FindSimulcastMaxLayers(width, height);
if (simulcast_layers > max_streams) {
std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
int width,
int height,
int max_bitrate_bps,
int max_qp,
int max_framerate,
bool is_screencast) {
size_t num_simulcast_layers;
if (is_screencast) {
num_simulcast_layers =
UseSimulcastScreenshare() ? kDefaultScreenshareSimulcastStreams : 1;
} else {
num_simulcast_layers = FindSimulcastMaxLayers(width, height);
}
if (num_simulcast_layers > max_streams) {
// If the number of SSRCs in the group differs from our target
// number of simulcast streams for current resolution, switch down
// to a resolution that matches our number of SSRCs.
if (!SlotSimulcastMaxResolution(max_streams, &width, &height)) {
return std::vector<webrtc::VideoStream>();
}
simulcast_layers = max_streams;
num_simulcast_layers = max_streams;
}
std::vector<webrtc::VideoStream> streams;
streams.resize(simulcast_layers);
streams.resize(num_simulcast_layers);
// Format width and height has to be divisible by |2 ^ number_streams - 1|.
width = NormalizeSimulcastSize(width, simulcast_layers);
height = NormalizeSimulcastSize(height, simulcast_layers);
if (!is_screencast) {
// Format width and height has to be divisible by |2 ^ number_streams - 1|.
width = NormalizeSimulcastSize(width, num_simulcast_layers);
height = NormalizeSimulcastSize(height, num_simulcast_layers);
}
// Add simulcast sub-streams from lower resolution to higher resolutions.
// Add simulcast streams, from highest resolution (|s| = number_streams -1)
// to lowest resolution at |s| = 0.
for (size_t s = simulcast_layers - 1;; --s) {
for (size_t s = num_simulcast_layers - 1;; --s) {
streams[s].width = width;
streams[s].height = height;
// TODO(pbos): Fill actual temporal-layer bitrate thresholds.
streams[s].temporal_layer_thresholds_bps.resize(
kDefaultConferenceNumberOfTemporalLayers[s] - 1);
streams[s].max_bitrate_bps =
FindSimulcastMaxBitrateBps(width, height, simulcast_layers);
streams[s].target_bitrate_bps =
FindSimulcastTargetBitrateBps(width, height, simulcast_layers);
streams[s].min_bitrate_bps =
FindSimulcastMinBitrateBps(width, height, simulcast_layers);
streams[s].max_qp = max_qp;
streams[s].max_framerate = max_framerate;
if (is_screencast && s == 0) {
ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
// For legacy screenshare in conference mode, tl0 and tl1 bitrates are
// piggybacked on the VideoCodec struct as target and max bitrates,
// respectively. See eg. webrtc::VP8EncoderImpl::SetRates().
streams[s].min_bitrate_bps = kMinVideoBitrateKbps * 1000;
streams[s].target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
streams[s].max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
streams[s].temporal_layer_thresholds_bps.clear();
streams[s].temporal_layer_thresholds_bps.push_back(
config.tl0_bitrate_kbps * 1000);
streams[s].max_framerate = 5;
} else {
streams[s].temporal_layer_thresholds_bps.resize(
kDefaultConferenceNumberOfTemporalLayers[s] - 1);
streams[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height);
streams[s].target_bitrate_bps =
FindSimulcastTargetBitrateBps(width, height);
streams[s].min_bitrate_bps = FindSimulcastMinBitrateBps(width, height);
streams[s].max_framerate = max_framerate;
}
width /= 2;
height /= 2;
if (s == 0) {
if (s == 0)
break;
}
}
// Spend additional bits to boost the max stream.
@ -230,6 +252,8 @@ static const int kScreenshareDefaultTl1BitrateKbps = 1000;
static const char* kScreencastLayerFieldTrialName =
"WebRTC-ScreenshareLayerRates";
static const char* kSimulcastScreenshareFieldTrialName =
"WebRTC-SimulcastScreenshare";
ScreenshareLayerConfig::ScreenshareLayerConfig(int tl0_bitrate, int tl1_bitrate)
: tl0_bitrate_kbps(tl0_bitrate), tl1_bitrate_kbps(tl1_bitrate) {
@ -272,4 +296,9 @@ bool ScreenshareLayerConfig::FromFieldTrialGroup(
return true;
}
bool UseSimulcastScreenshare() {
return webrtc::field_trial::FindFullName(
kSimulcastScreenshareFieldTrialName) == "Enabled";
}
} // namespace cricket

7
webrtc/media/engine/simulcast.h
View File

@ -19,6 +19,7 @@
namespace cricket {
struct StreamParams;
// TODO(sprang): Remove this, as we're moving away from temporal layer mode.
// Config for use with screen cast when temporal layers are enabled.
struct ScreenshareLayerConfig {
public:
@ -45,12 +46,16 @@ int GetTotalMaxBitrateBps(const std::vector<webrtc::VideoStream>& streams);
void GetSimulcastSsrcs(const StreamParams& sp, std::vector<uint32_t>* ssrcs);
// Get simulcast settings.
// TODO(sprang): Remove default parameter when it's not longer referenced.
std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_streams,
int width,
int height,
int max_bitrate_bps,
int max_qp,
int max_framerate);
int max_framerate,
bool is_screencast = false);
bool UseSimulcastScreenshare();
} // namespace cricket

50
webrtc/media/engine/webrtcvideoengine2.cc
View File

@ -301,11 +301,16 @@ class EncoderStreamFactory
int width,
int height,
const webrtc::VideoEncoderConfig& encoder_config) override {
RTC_DCHECK(encoder_config.number_of_streams > 1 ? !is_screencast_ : true);
if (encoder_config.number_of_streams > 1) {
if (is_screencast_ &&
(!conference_mode_ || !cricket::UseSimulcastScreenshare())) {
RTC_DCHECK_EQ(1, encoder_config.number_of_streams);
}
if (encoder_config.number_of_streams > 1 ||
(CodecNamesEq(codec_name_, kVp8CodecName) && is_screencast_ &&
conference_mode_)) {
return GetSimulcastConfig(encoder_config.number_of_streams, width, height,
encoder_config.max_bitrate_bps, max_qp_,
max_framerate_);
max_framerate_, is_screencast_);
}
// For unset max bitrates set default bitrate for non-simulcast.
@ -322,20 +327,6 @@ class EncoderStreamFactory
stream.target_bitrate_bps = stream.max_bitrate_bps = max_bitrate_bps;
stream.max_qp = max_qp_;
// Conference mode screencast uses 2 temporal layers split at 100kbit.
if (conference_mode_ && is_screencast_) {
ScreenshareLayerConfig config = ScreenshareLayerConfig::GetDefault();
// For screenshare in conference mode, tl0 and tl1 bitrates are
// piggybacked
// on the VideoCodec struct as target and max bitrates, respectively.
// See eg. webrtc::VP8EncoderImpl::SetRates().
stream.target_bitrate_bps = config.tl0_bitrate_kbps * 1000;
stream.max_bitrate_bps = config.tl1_bitrate_kbps * 1000;
stream.temporal_layer_thresholds_bps.clear();
stream.temporal_layer_thresholds_bps.push_back(config.tl0_bitrate_kbps *
1000);
}
if (CodecNamesEq(codec_name_, kVp9CodecName) && !is_screencast_) {
stream.temporal_layer_thresholds_bps.resize(
GetDefaultVp9TemporalLayers() - 1);
@ -1551,6 +1542,7 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::WebRtcVideoSendStream(
enable_cpu_overuse_detection_(enable_cpu_overuse_detection),
source_(nullptr),
external_encoder_factory_(external_encoder_factory),
internal_encoder_factory_(new InternalEncoderFactory()),
stream_(nullptr),
encoder_sink_(nullptr),
parameters_(std::move(config), options, max_bitrate_bps, codec_settings),
@ -1678,10 +1670,20 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoder(
}
// Try creating internal encoder.
InternalEncoderFactory internal_encoder_factory;
if (FindMatchingCodec(internal_encoder_factory.supported_codecs(), codec)) {
return AllocatedEncoder(internal_encoder_factory.CreateVideoEncoder(codec),
codec, false /* is_external */);
if (FindMatchingCodec(internal_encoder_factory_->supported_codecs(), codec)) {
if (parameters_.encoder_config.content_type ==
webrtc::VideoEncoderConfig::ContentType::kScreen &&
parameters_.conference_mode && UseSimulcastScreenshare()) {
// TODO(sprang): Remove this adapter once libvpx supports simulcast with
// same-resolution substreams.
WebRtcSimulcastEncoderFactory adapter_factory(
internal_encoder_factory_.get());
return AllocatedEncoder(adapter_factory.CreateVideoEncoder(codec), codec,
false /* is_external */);
}
return AllocatedEncoder(
internal_encoder_factory_->CreateVideoEncoder(codec), codec,
false /* is_external */);
}
// This shouldn't happen, we should not be trying to create something we don't
@ -1858,9 +1860,11 @@ WebRtcVideoChannel2::WebRtcVideoSendStream::CreateVideoEncoderConfig(
// By default, the stream count for the codec configuration should match the
// number of negotiated ssrcs. But if the codec is blacklisted for simulcast
// or a screencast, only configure a single stream.
// or a screencast (and not in simulcast screenshare experiment), only
// configure a single stream.
encoder_config.number_of_streams = parameters_.config.rtp.ssrcs.size();
if (IsCodecBlacklistedForSimulcast(codec.name) || is_screencast) {
if (IsCodecBlacklistedForSimulcast(codec.name) ||
(is_screencast && !UseSimulcastScreenshare())) {
encoder_config.number_of_streams = 1;
}

2
webrtc/media/engine/webrtcvideoengine2.h
View File

@ -330,6 +330,8 @@ class WebRtcVideoChannel2 : public VideoMediaChannel, public webrtc::Transport {
ACCESS_ON(&thread_checker_);
WebRtcVideoEncoderFactory* const external_encoder_factory_
ACCESS_ON(&thread_checker_);
const std::unique_ptr<WebRtcVideoEncoderFactory> internal_encoder_factory_
ACCESS_ON(&thread_checker_);
webrtc::VideoSendStream* stream_ ACCESS_ON(&thread_checker_);
rtc::VideoSinkInterface<webrtc::VideoFrame>* encoder_sink_

74
webrtc/media/engine/webrtcvideoengine2_unittest.cc
View File

@ -21,6 +21,7 @@
#include "webrtc/media/base/mediaconstants.h"
#include "webrtc/media/base/testutils.h"
#include "webrtc/media/base/videoengine_unittest.h"
#include "webrtc/media/engine/constants.h"
#include "webrtc/media/engine/fakewebrtccall.h"
#include "webrtc/media/engine/fakewebrtcvideoengine.h"
#include "webrtc/media/engine/simulcast.h"
@ -3960,7 +3961,7 @@ TEST_F(WebRtcVideoChannel2Test, ConfiguresLocalSsrcOnExistingReceivers) {
class WebRtcVideoChannel2SimulcastTest : public testing::Test {
public:
WebRtcVideoChannel2SimulcastTest()
: fake_call_(webrtc::Call::Config(&event_log_)) {}
: fake_call_(webrtc::Call::Config(&event_log_)), last_ssrc_(0) {}
void SetUp() override {
engine_.Init();
@ -3975,9 +3976,16 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test {
int capture_width,
int capture_height,
size_t num_configured_streams,
size_t expected_num_streams) {
size_t expected_num_streams,
bool screenshare,
bool conference_mode) {
cricket::VideoSendParameters parameters;
VideoOptions options;
parameters.codecs.push_back(codec);
parameters.conference_mode = conference_mode;
if (screenshare) {
options.is_screencast = rtc::Optional<bool>(screenshare);
}
ASSERT_TRUE(channel_->SetSendParameters(parameters));
std::vector<uint32_t> ssrcs = MAKE_VECTOR(kSsrcs3);
@ -3990,7 +3998,7 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test {
// expected simulcast layers.
cricket::FakeVideoCapturer capturer;
EXPECT_TRUE(
channel_->SetVideoSend(ssrcs.front(), true, nullptr, &capturer));
channel_->SetVideoSend(ssrcs.front(), true, &options, &capturer));
EXPECT_EQ(cricket::CS_RUNNING, capturer.Start(cricket::VideoFormat(
capture_width, capture_height,
cricket::VideoFormat::FpsToInterval(30),
@ -4001,9 +4009,32 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test {
std::vector<webrtc::VideoStream> video_streams = stream->GetVideoStreams();
ASSERT_EQ(expected_num_streams, video_streams.size());
std::vector<webrtc::VideoStream> expected_streams = GetSimulcastConfig(
num_configured_streams, capture_width, capture_height, 0, kDefaultQpMax,
kDefaultVideoMaxFramerate);
std::vector<webrtc::VideoStream> expected_streams;
if (conference_mode) {
expected_streams = GetSimulcastConfig(
num_configured_streams, capture_width, capture_height, 0,
kDefaultQpMax, kDefaultVideoMaxFramerate, screenshare);
} else {
webrtc::VideoStream stream;
stream.width = capture_width;
stream.height = capture_height;
stream.max_framerate = kDefaultVideoMaxFramerate;
stream.min_bitrate_bps = cricket::kMinVideoBitrateKbps * 1000;
int max_bitrate_kbps;
if (capture_width * capture_height <= 320 * 240) {
max_bitrate_kbps = 600;
} else if (capture_width * capture_height <= 640 * 480) {
max_bitrate_kbps = 1700;
} else if (capture_width * capture_height <= 960 * 540) {
max_bitrate_kbps = 2000;
} else {
max_bitrate_kbps = 2500;
}
stream.target_bitrate_bps = stream.max_bitrate_bps =
max_bitrate_kbps * 1000;
stream.max_qp = kDefaultQpMax;
expected_streams.push_back(stream);
}
ASSERT_EQ(expected_streams.size(), video_streams.size());
@ -4032,7 +4063,8 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test {
EXPECT_GT(video_streams[i].max_qp, 0);
EXPECT_EQ(expected_streams[i].max_qp, video_streams[i].max_qp);
EXPECT_FALSE(expected_streams[i].temporal_layer_thresholds_bps.empty());
EXPECT_EQ(!conference_mode,
expected_streams[i].temporal_layer_thresholds_bps.empty());
EXPECT_EQ(expected_streams[i].temporal_layer_thresholds_bps,
video_streams[i].temporal_layer_thresholds_bps);
@ -4086,15 +4118,37 @@ class WebRtcVideoChannel2SimulcastTest : public testing::Test {
};
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith2SimulcastStreams) {
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2);
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 640, 360, 2, 2, false,
true);
}
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWith3SimulcastStreams) {
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3);
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 3, false,
true);
}
// Test that we normalize send codec format size in simulcast.
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsWithOddSizeInSimulcast) {
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2);
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 541, 271, 2, 2, false,
true);
}
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForScreenshare) {
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true,
false);
}
TEST_F(WebRtcVideoChannel2SimulcastTest,
SetSendCodecsForConferenceModeScreenshare) {
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 1, true,
true);
}
TEST_F(WebRtcVideoChannel2SimulcastTest, SetSendCodecsForSimulcastScreenshare) {
webrtc::test::ScopedFieldTrials override_field_trials_(
"WebRTC-SimulcastScreenshare/Enabled/");
VerifySimulcastSettings(cricket::VideoCodec("VP8"), 1280, 720, 3, 2, true,
true);
}
} // namespace cricket

1
webrtc/modules/video_coding/BUILD.gn
View File

@ -371,6 +371,7 @@ if (rtc_include_tests) {
"utility/moving_average_unittest.cc",
"utility/quality_scaler_unittest.cc",
"utility/simulcast_rate_allocator_unittest.cc",
"video_codec_initializer_unittest.cc",
"video_coding_robustness_unittest.cc",
"video_packet_buffer_unittest.cc",
"video_receiver_unittest.cc",

6
webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc
View File

@ -94,11 +94,11 @@ class RealTimeTemporalLayers : public TemporalLayers {
timestamp_(0),
last_base_layer_sync_(0),
layer_ids_length_(0),
layer_ids_(NULL),
layer_ids_(nullptr),
encode_flags_length_(0),
encode_flags_(NULL) {
encode_flags_(nullptr) {
RTC_CHECK_GE(max_temporal_layers_, 1);
RTC_CHECK_GE(max_temporal_layers_, 3);
RTC_CHECK_LE(max_temporal_layers_, 3);
}
virtual ~RealTimeTemporalLayers() {}

19
webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
View File

@ -30,6 +30,8 @@ static const int kQpDeltaThresholdForSync = 8;
const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
// Since this is TL0 we only allow updating and predicting from the LAST
// reference frame.
const int ScreenshareLayers::kTl0Flags =
@ -55,8 +57,14 @@ webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create(
int simulcast_id,
int num_temporal_layers,
uint8_t initial_tl0_pic_idx) const {
webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers(
num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock());
webrtc::TemporalLayers* tl;
if (simulcast_id == 0) {
tl = new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
webrtc::Clock::GetRealTimeClock());
} else {
RealTimeTemporalLayersFactory rt_tl_factory;
tl = rt_tl_factory.Create(simulcast_id, num_temporal_layers, rand());
}
if (listener_)
listener_->OnTemporalLayersCreated(simulcast_id, tl);
return tl;
@ -66,7 +74,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
uint8_t initial_tl0_pic_idx,
Clock* clock)
: clock_(clock),
number_of_temporal_layers_(num_temporal_layers),
number_of_temporal_layers_(
std::min(kMaxNumTemporalLayers, num_temporal_layers)),
last_base_layer_sync_(false),
tl0_pic_idx_(initial_tl0_pic_idx),
active_layer_(-1),
@ -78,8 +87,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
max_debt_bytes_(0),
encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
bitrate_updated_(false) {
RTC_CHECK_GT(num_temporal_layers, 0);
RTC_CHECK_LE(num_temporal_layers, 2);
RTC_CHECK_GT(number_of_temporal_layers_, 0);
RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
}
ScreenshareLayers::~ScreenshareLayers() {

2
webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
View File

@ -84,7 +84,7 @@ class ScreenshareLayers : public TemporalLayers {
RateStatistics encode_framerate_;
bool bitrate_updated_;
static const int kMaxNumTemporalLayers = 2;
static constexpr int kMaxNumTemporalLayers = 2;
struct TemporalLayer {
TemporalLayer()
: state(State::kNormal),

23
webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc
View File

@ -106,23 +106,24 @@ BitrateAllocation SimulcastRateAllocator::GetAllocation(
const int num_temporal_streams = std::max<uint8_t>(
1, codec_.numberOfSimulcastStreams == 0
? codec_.VP8().numberOfTemporalLayers
: codec_.simulcastStream[0].numberOfTemporalLayers);
: codec_.simulcastStream[simulcast_id].numberOfTemporalLayers);
uint32_t max_bitrate_kbps;
if (num_spatial_streams == 1) {
max_bitrate_kbps = codec_.maxBitrate;
// TODO(holmer): This is a temporary hack for screensharing, where we
// Legacy temporal-layered only screenshare, or simulcast screenshare
// with legacy mode for simulcast stream 0.
if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 &&
((num_spatial_streams == 1 && num_temporal_streams == 2) || // Legacy.
(num_spatial_streams > 1 && simulcast_id == 0))) { // Simulcast.
// TODO(holmer): This is a "temporary" hack for screensharing, where we
// interpret the startBitrate as the encoder target bitrate. This is
// to allow for a different max bitrate, so if the codec can't meet
// the target we still allow it to overshoot up to the max before dropping
// frames. This hack should be improved.
if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 &&
num_temporal_streams == 2) {
int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps);
max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps);
target_bitrate_kbps = tl0_bitrate;
}
int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps);
max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps);
target_bitrate_kbps = tl0_bitrate;
} else if (num_spatial_streams == 1) {
max_bitrate_kbps = codec_.maxBitrate;
} else {
max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate;
}

15
webrtc/modules/video_coding/video_codec_initializer.cc
View File

@ -38,8 +38,9 @@ bool VideoCodecInitializer::SetupCodec(
case kVideoCodecVP8: {
if (!codec->VP8()->tl_factory) {
if (codec->mode == kScreensharing &&
codec->numberOfSimulcastStreams == 1 &&
codec->VP8()->numberOfTemporalLayers == 2) {
(codec->numberOfSimulcastStreams > 1 ||
(codec->numberOfSimulcastStreams == 1 &&
codec->VP8()->numberOfTemporalLayers == 2))) {
// Conference mode temporal layering for screen content.
tl_factory.reset(new ScreenshareTemporalLayersFactory());
} else {
@ -102,7 +103,7 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
break;
case VideoEncoderConfig::ContentType::kScreen:
video_codec.mode = kScreensharing;
if (streams.size() == 1 &&
if (streams.size() >= 1 &&
streams[0].temporal_layer_thresholds_bps.size() == 1) {
video_codec.targetBitrate =
streams[0].temporal_layer_thresholds_bps[0] / 1000;
@ -180,8 +181,12 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
RTC_DCHECK_GT(streams[i].width, 0);
RTC_DCHECK_GT(streams[i].height, 0);
RTC_DCHECK_GT(streams[i].max_framerate, 0);
// Different framerates not supported per stream at the moment.
RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
// Different framerates not supported per stream at the moment, unless it's
// screenshare where there is an exception and a simulcast encoder adapter,
// which supports different framerates, is used instead.
if (config.content_type != VideoEncoderConfig::ContentType::kScreen) {
RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
}
RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);

217
webrtc/modules/video_coding/video_codec_initializer_unittest.cc Normal file
View File

@ -0,0 +1,217 @@
/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/common_video/include/video_bitrate_allocator.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
#include "webrtc/modules/video_coding/include/video_codec_initializer.h"
#include "webrtc/test/gtest.h"
#include "webrtc/video_encoder.h"
namespace webrtc {
namespace {
static const char* kVp8PayloadName = "VP8";
static const int kVp8PayloadType = 100;
static const int kDefaultWidth = 1280;
static const int kDefaultHeight = 720;
static const int kDefaultFrameRate = 30;
static const uint32_t kDefaultMinBitrateBps = 60000;
static const uint32_t kDefaultTargetBitrateBps = 2000000;
static const uint32_t kDefaultMaxBitrateBps = 2000000;
static const uint32_t kDefaultMinTransmitBitrateBps = 400000;
static const int kDefaultMaxQp = 48;
static const uint32_t kScreenshareTl0BitrateBps = 100000;
static const uint32_t kScreenshareCodecTargetBitrateBps = 200000;
static const uint32_t kScreenshareDefaultFramerate = 5;
// Bitrates for the temporal layers of the higher screenshare simulcast stream.
static const uint32_t kHighScreenshareTl0Bps = 800000;
static const uint32_t kHighScreenshareTl1Bps = 1200000;
} // namespace
/*
* static bool SetupCodec(
const VideoEncoderConfig& config,
const VideoSendStream::Config::EncoderSettings settings,
const std::vector<VideoStream>& streams,
bool nack_enabled,
VideoCodec* codec,
std::unique_ptr<VideoBitrateAllocator>* bitrate_allocator);
// Create a bitrate allocator for the specified codec. |tl_factory| is
// optional, if it is populated, ownership of that instance will be
// transferred to the VideoBitrateAllocator instance.
static std::unique_ptr<VideoBitrateAllocator> CreateBitrateAllocator(
const VideoCodec& codec,
std::unique_ptr<TemporalLayersFactory> tl_factory);
*/
// TODO(sprang): Extend coverage to handle the rest of the codec initializer.
class VideoCodecInitializerTest : public ::testing::Test {
public:
VideoCodecInitializerTest() : nack_enabled_(false) {}
virtual ~VideoCodecInitializerTest() {}
protected:
void SetUpFor(VideoCodecType type,
int num_spatial_streams,
int num_temporal_streams,
bool screenshare) {
config_ = VideoEncoderConfig();
if (screenshare) {
config_.min_transmit_bitrate_bps = kDefaultMinTransmitBitrateBps;
config_.content_type = VideoEncoderConfig::ContentType::kScreen;
}
if (type == VideoCodecType::kVideoCodecVP8) {
config_.number_of_streams = num_spatial_streams;
VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
vp8_settings.numberOfTemporalLayers = num_temporal_streams;
config_.encoder_specific_settings = new rtc::RefCountedObject<
webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
settings_.payload_name = kVp8PayloadName;
settings_.payload_type = kVp8PayloadType;
} else {
ADD_FAILURE() << "Unexpected codec type: " << type;
}
}
bool InitializeCodec() {
codec_out_ = VideoCodec();
bitrate_allocator_out_.reset();
temporal_layers_.clear();
if (!VideoCodecInitializer::SetupCodec(config_, settings_, streams_,
nack_enabled_, &codec_out_,
&bitrate_allocator_out_)) {
return false;
}
// Make sure temporal layers instances have been created.
if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) {
if (!codec_out_.VP8()->tl_factory)
return false;
for (int i = 0; i < codec_out_.numberOfSimulcastStreams; ++i) {
temporal_layers_.emplace_back(codec_out_.VP8()->tl_factory->Create(
i, streams_[i].temporal_layer_thresholds_bps.size() + 1, 0));
}
}
return true;
}
VideoStream DefaultStream() {
VideoStream stream;
stream.width = kDefaultWidth;
stream.height = kDefaultHeight;
stream.max_framerate = kDefaultFrameRate;
stream.min_bitrate_bps = kDefaultMinBitrateBps;
stream.target_bitrate_bps = kDefaultTargetBitrateBps;
stream.max_bitrate_bps = kDefaultMaxBitrateBps;
stream.max_qp = kDefaultMaxQp;
return stream;
}
VideoStream DefaultScreenshareStream() {
VideoStream stream = DefaultStream();
stream.min_bitrate_bps = 30000;
stream.target_bitrate_bps = kScreenshareTl0BitrateBps;
stream.max_bitrate_bps = 1000000;
stream.max_framerate = kScreenshareDefaultFramerate;
stream.temporal_layer_thresholds_bps.push_back(kScreenshareTl0BitrateBps);
return stream;
}
// Input settings.
VideoEncoderConfig config_;
VideoSendStream::Config::EncoderSettings settings_;
std::vector<VideoStream> streams_;
bool nack_enabled_;
// Output.
VideoCodec codec_out_;
std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_out_;
std::vector<std::unique_ptr<TemporalLayers>> temporal_layers_;
};
TEST_F(VideoCodecInitializerTest, SingleStreamVp8Screenshare) {
SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 1, true);
streams_.push_back(DefaultStream());
EXPECT_TRUE(InitializeCodec());
BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
kDefaultTargetBitrateBps, kDefaultFrameRate);
EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
EXPECT_EQ(kDefaultTargetBitrateBps, bitrate_allocation.get_sum_bps());
}
TEST_F(VideoCodecInitializerTest, TemporalLayeredVp8Screenshare) {
SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 2, true);
streams_.push_back(DefaultScreenshareStream());
EXPECT_TRUE(InitializeCodec());
EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
EXPECT_EQ(2u, codec_out_.VP8()->numberOfTemporalLayers);
BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
kScreenshareCodecTargetBitrateBps, kScreenshareDefaultFramerate);
EXPECT_EQ(kScreenshareCodecTargetBitrateBps,
bitrate_allocation.get_sum_bps());
EXPECT_EQ(kScreenshareTl0BitrateBps, bitrate_allocation.GetBitrate(0, 0));
}
TEST_F(VideoCodecInitializerTest, SimlucastVp8Screenshare) {
SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 1, true);
streams_.push_back(DefaultScreenshareStream());
VideoStream video_stream = DefaultStream();
video_stream.max_framerate = kScreenshareDefaultFramerate;
streams_.push_back(video_stream);
EXPECT_TRUE(InitializeCodec());
EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
const uint32_t max_bitrate_bps =
streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
max_bitrate_bps, kScreenshareDefaultFramerate);
EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
bitrate_allocation.GetSpatialLayerSum(0));
EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
bitrate_allocation.GetSpatialLayerSum(1));
}
TEST_F(VideoCodecInitializerTest, HighFpsSimlucastVp8Screenshare) {
// Two simulcast streams, the lower one using legacy settings (two temporal
// streams, 5fps), the higher one using 3 temporal streams and 30fps.
SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 3, true);
streams_.push_back(DefaultScreenshareStream());
VideoStream video_stream = DefaultStream();
video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl0Bps);
video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl1Bps);
streams_.push_back(video_stream);
EXPECT_TRUE(InitializeCodec());
EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
EXPECT_EQ(3u, codec_out_.VP8()->numberOfTemporalLayers);
const uint32_t max_bitrate_bps =
streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
BitrateAllocation bitrate_allocation =
bitrate_allocator_out_->GetAllocation(max_bitrate_bps, kDefaultFrameRate);
EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
bitrate_allocation.GetSpatialLayerSum(0));
EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
bitrate_allocation.GetSpatialLayerSum(1));
EXPECT_EQ(kHighScreenshareTl0Bps, bitrate_allocation.GetBitrate(1, 0));
EXPECT_EQ(kHighScreenshareTl1Bps - kHighScreenshareTl0Bps,
bitrate_allocation.GetBitrate(1, 1));
}
} // namespace webrtc