Generalize SimulcastEncoderAdapter, use for H264 & VP8.

* Move SimulcastEncoderAdapter out under modules/video_coding
* Move SimulcastRateAllocator back out to modules/video_coding/utility
* Move TemporalLayers and ScreenshareLayers to modules/video_coding/utility
* Move any VP8 specific code - such as temporal layer bitrate budgeting -
  under codec type dependent conditionals.
* Plumb the simulcast index for H264 in the codec specific and RTP format data structures.

TBR=sprang@webrtc.org,stefan@webrtc.org,titovartem@webrtc.org

Bug: webrtc:5840
Change-Id: I2d3b130622dd7ceec5528f3ab6c46f109e6bafb8
Reviewed-on: https://webrtc-review.googlesource.com/84743
Commit-Queue: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23715}

AUTHORS

@@ -61,6 +61,7 @@ Yura Yaroshevich <yura.yaroshevich@gmail.com>
 Hans Knoechel <hans@hans-knoechel.de>
 Korniltsev Anatoly <korniltsev.anatoly@gmail.com>
 Todd Wong <todd.wong.ndq@gmail.com>
+Sergio Garcia Murillo <sergio.garcia.murillo@gmail.com>
 Maxim Pavlov <pavllovmax@gmail.com>
 Yusuke Suzuki <utatane.tea@gmail.com>
 Piasy Xu <xz4215@gmail.com>
@@ -89,3 +90,5 @@ Vonage Holdings Corp. <*@vonage.com>
 Wire Swiss GmbH <*@wire.com>
 Miguel Paris <mparisdiaz@gmail.com>
 Vewd Software AS <*@vewd.com>
+Highfive, Inc. <*@highfive.com>
+CoSMo Software Consulting, Pte Ltd <*@cosmosoftware.io>

api/test/create_simulcast_test_fixture.cc

@@ -14,7 +14,7 @@
 #include <utility>
 
 #include "api/test/simulcast_test_fixture.h"
-#include "modules/video_coding/codecs/vp8/simulcast_test_fixture_impl.h"
+#include "modules/video_coding/utility/simulcast_test_fixture_impl.h"
 #include "rtc_base/ptr_util.h"
 
 namespace webrtc {
@@ -22,9 +22,11 @@ namespace test {
 
 std::unique_ptr<SimulcastTestFixture> CreateSimulcastTestFixture(
     std::unique_ptr<VideoEncoderFactory> encoder_factory,
-    std::unique_ptr<VideoDecoderFactory> decoder_factory) {
+    std::unique_ptr<VideoDecoderFactory> decoder_factory,
+    SdpVideoFormat video_format) {
   return rtc::MakeUnique<SimulcastTestFixtureImpl>(std::move(encoder_factory),
-                                                   std::move(decoder_factory));
+                                                   std::move(decoder_factory),
+                                                   video_format);
 }
 
 }  // namespace test

api/test/create_simulcast_test_fixture.h

@@ -14,6 +14,7 @@
 #include <memory>
 
 #include "api/test/simulcast_test_fixture.h"
+#include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_decoder_factory.h"
 #include "api/video_codecs/video_encoder_factory.h"
 
@@ -22,7 +23,8 @@ namespace test {
 
 std::unique_ptr<SimulcastTestFixture> CreateSimulcastTestFixture(
     std::unique_ptr<VideoEncoderFactory> encoder_factory,
-    std::unique_ptr<VideoDecoderFactory> decoder_factory);
+    std::unique_ptr<VideoDecoderFactory> decoder_factory,
+    SdpVideoFormat video_format);
 
 }  // namespace test
 }  // namespace webrtc

api/video_codecs/test/BUILD.gn

@@ -22,8 +22,8 @@ if (rtc_include_tests) {
       "..:rtc_software_fallback_wrappers",
       "..:video_codecs_api",
       "../../../modules/video_coding:video_codec_interface",
+      "../../../modules/video_coding:video_coding_utility",
       "../../../modules/video_coding:webrtc_vp8",
-      "../../../modules/video_coding:webrtc_vp8_helpers",
       "../../../rtc_base:checks",
       "../../../rtc_base:rtc_base_tests_utils",
       "../../../system_wrappers:metrics_default",

api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc

@@ -15,10 +15,10 @@
 #include "api/video/i420_buffer.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
 #include "modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/include/video_error_codes.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/fakeclock.h"
 #include "test/field_trial.h"

media/BUILD.gn

@@ -191,10 +191,10 @@ rtc_static_library("rtc_internal_video_codecs") {
     "../api/video_codecs:video_codecs_api",
     "../call:call_interfaces",
     "../call:video_stream_api",
+    "../modules/video_coding:video_coding_utility",
     "../modules/video_coding:webrtc_h264",
     "../modules/video_coding:webrtc_multiplex",
     "../modules/video_coding:webrtc_vp8",
-    "../modules/video_coding:webrtc_vp8_helpers",
     "../modules/video_coding:webrtc_vp9",
     "../rtc_base:checks",
     "../rtc_base:rtc_base_approved",
@@ -218,6 +218,7 @@ rtc_static_library("rtc_audio_video") {
     "../modules/audio_processing/aec_dump:aec_dump",
     "../modules/video_coding:video_codec_interface",
     "../modules/video_coding:video_coding",
+    "../modules/video_coding:video_coding_utility",
     "../rtc_base:audio_format_to_string",
     "../rtc_base:base64",
     "../rtc_base:checks",
@@ -304,7 +305,6 @@ rtc_static_library("rtc_audio_video") {
     "../modules/audio_mixer:audio_mixer_impl",
     "../modules/audio_processing:audio_processing",
     "../modules/video_capture:video_capture_module",
-    "../modules/video_coding:webrtc_vp8_helpers",
     "../pc:rtc_pc_base",
     "../rtc_base:rtc_base",
     "../rtc_base:rtc_task_queue",
@@ -474,6 +474,7 @@ if (rtc_include_tests) {
       "../api/video:video_frame_i420",
       "../modules/audio_processing:mocks",
       "../modules/video_coding:video_codec_interface",
+      "../modules/video_coding:webrtc_vp8",
       "../pc:rtc_pc",
       "../pc:rtc_pc_base",
       "../rtc_base:checks",
@@ -569,7 +570,6 @@ if (rtc_include_tests) {
       "../modules/audio_device:mock_audio_device",
       "../modules/audio_processing:audio_processing",
       "../modules/video_coding:simulcast_test_fixture_impl",
-      "../modules/video_coding:webrtc_vp8_helpers",
       "../p2p:p2p_test_utils",
       "../rtc_base:rtc_base",
       "../rtc_base:rtc_base_approved",

media/DEPS

@@ -10,6 +10,7 @@ include_rules = [
   "+modules/rtp_rtcp",
   "+modules/video_capture",
   "+modules/video_coding",
+  "+modules/video_coding/utility",
   "+p2p",
   "+pc",
   "+sound",

media/engine/fakewebrtccall.cc

@@ -21,16 +21,17 @@
 
 namespace cricket {
 FakeAudioSendStream::FakeAudioSendStream(
-    int id,
+    int id, const webrtc::AudioSendStream::Config& config)
-    const webrtc::AudioSendStream::Config& config)
+    : id_(id), config_(config) {
-    : id_(id), config_(config) {}
+}
 
 void FakeAudioSendStream::Reconfigure(
     const webrtc::AudioSendStream::Config& config) {
   config_ = config;
 }
 
-const webrtc::AudioSendStream::Config& FakeAudioSendStream::GetConfig() const {
+const webrtc::AudioSendStream::Config&
+    FakeAudioSendStream::GetConfig() const {
   return config_;
 }
 
@@ -45,8 +46,7 @@ FakeAudioSendStream::GetLatestTelephoneEvent() const {
 }
 
 bool FakeAudioSendStream::SendTelephoneEvent(int payload_type,
-                                             int payload_frequency,
+                                             int payload_frequency, int event,
-                                             int event,
                                              int duration_ms) {
   latest_telephone_event_.payload_type = payload_type;
   latest_telephone_event_.payload_frequency = payload_frequency;
@@ -69,12 +69,12 @@ webrtc::AudioSendStream::Stats FakeAudioSendStream::GetStats(
 }
 
 FakeAudioReceiveStream::FakeAudioReceiveStream(
-    int id,
+    int id, const webrtc::AudioReceiveStream::Config& config)
-    const webrtc::AudioReceiveStream::Config& config)
+    : id_(id), config_(config) {
-    : id_(id), config_(config) {}
+}
 
-const webrtc::AudioReceiveStream::Config& FakeAudioReceiveStream::GetConfig()
+const webrtc::AudioReceiveStream::Config&
-    const {
+    FakeAudioReceiveStream::GetConfig() const {
   return config_;
 }
 
@@ -156,7 +156,7 @@ bool FakeVideoSendStream::GetVp8Settings(
     return false;
   }
 
-  *settings = vpx_settings_.vp8;
+  *settings = codec_specific_settings_.vp8;
   return true;
 }
 
@@ -166,7 +166,17 @@ bool FakeVideoSendStream::GetVp9Settings(
     return false;
   }
 
-  *settings = vpx_settings_.vp9;
+  *settings = codec_specific_settings_.vp9;
+  return true;
+}
+
+bool FakeVideoSendStream::GetH264Settings(
+    webrtc::VideoCodecH264* settings) const {
+  if (!codec_settings_set_) {
+    return false;
+  }
+
+  *settings = codec_specific_settings_.h264;
   return true;
 }
 
@@ -189,7 +199,8 @@ int64_t FakeVideoSendStream::GetLastTimestamp() const {
 
 void FakeVideoSendStream::OnFrame(const webrtc::VideoFrame& frame) {
   ++num_swapped_frames_;
-  if (!last_frame_ || frame.width() != last_frame_->width() ||
+  if (!last_frame_ ||
+      frame.width() != last_frame_->width() ||
       frame.height() != last_frame_->height() ||
       frame.rotation() != last_frame_->rotation()) {
     video_streams_ = encoder_config_.video_stream_factory->CreateEncoderStreams(
@@ -229,15 +240,22 @@ void FakeVideoSendStream::ReconfigureVideoEncoder(
     const unsigned char num_temporal_layers = static_cast<unsigned char>(
         video_streams_.back().num_temporal_layers.value_or(1));
     if (config_.rtp.payload_name == "VP8") {
-      config.encoder_specific_settings->FillVideoCodecVp8(&vpx_settings_.vp8);
+      config.encoder_specific_settings->FillVideoCodecVp8(
+          &codec_specific_settings_.vp8);
       if (!video_streams_.empty()) {
-        vpx_settings_.vp8.numberOfTemporalLayers = num_temporal_layers;
+        codec_specific_settings_.vp8.numberOfTemporalLayers =
+            num_temporal_layers;
       }
     } else if (config_.rtp.payload_name == "VP9") {
-      config.encoder_specific_settings->FillVideoCodecVp9(&vpx_settings_.vp9);
+      config.encoder_specific_settings->FillVideoCodecVp9(
+          &codec_specific_settings_.vp9);
       if (!video_streams_.empty()) {
-        vpx_settings_.vp9.numberOfTemporalLayers = num_temporal_layers;
+        codec_specific_settings_.vp9.numberOfTemporalLayers =
+            num_temporal_layers;
       }
+    } else if (config_.rtp.payload_name == "H264") {
+      config.encoder_specific_settings->FillVideoCodecH264(
+          &codec_specific_settings_.h264);
     } else {
       ADD_FAILURE() << "Unsupported encoder payload: "
                     << config_.rtp.payload_name;
@@ -456,15 +474,16 @@ webrtc::NetworkState FakeCall::GetNetworkState(webrtc::MediaType media) const {
 
 webrtc::AudioSendStream* FakeCall::CreateAudioSendStream(
     const webrtc::AudioSendStream::Config& config) {
-  FakeAudioSendStream* fake_stream =
+  FakeAudioSendStream* fake_stream = new FakeAudioSendStream(next_stream_id_++,
-      new FakeAudioSendStream(next_stream_id_++, config);
+                                                             config);
   audio_send_streams_.push_back(fake_stream);
   ++num_created_send_streams_;
   return fake_stream;
 }
 
 void FakeCall::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) {
-  auto it = std::find(audio_send_streams_.begin(), audio_send_streams_.end(),
+  auto it = std::find(audio_send_streams_.begin(),
+                      audio_send_streams_.end(),
                       static_cast<FakeAudioSendStream*>(send_stream));
   if (it == audio_send_streams_.end()) {
     ADD_FAILURE() << "DestroyAudioSendStream called with unknown parameter.";
@@ -476,16 +495,16 @@ void FakeCall::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) {
 
 webrtc::AudioReceiveStream* FakeCall::CreateAudioReceiveStream(
     const webrtc::AudioReceiveStream::Config& config) {
-  audio_receive_streams_.push_back(
+  audio_receive_streams_.push_back(new FakeAudioReceiveStream(next_stream_id_++,
-      new FakeAudioReceiveStream(next_stream_id_++, config));
+                                                              config));
   ++num_created_receive_streams_;
   return audio_receive_streams_.back();
 }
 
 void FakeCall::DestroyAudioReceiveStream(
     webrtc::AudioReceiveStream* receive_stream) {
-  auto it =
+  auto it = std::find(audio_receive_streams_.begin(),
-      std::find(audio_receive_streams_.begin(), audio_receive_streams_.end(),
+                      audio_receive_streams_.end(),
                       static_cast<FakeAudioReceiveStream*>(receive_stream));
   if (it == audio_receive_streams_.end()) {
     ADD_FAILURE() << "DestroyAudioReceiveStream called with unknown parameter.";
@@ -506,7 +525,8 @@ webrtc::VideoSendStream* FakeCall::CreateVideoSendStream(
 }
 
 void FakeCall::DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) {
-  auto it = std::find(video_send_streams_.begin(), video_send_streams_.end(),
+  auto it = std::find(video_send_streams_.begin(),
+                      video_send_streams_.end(),
                       static_cast<FakeVideoSendStream*>(send_stream));
   if (it == video_send_streams_.end()) {
     ADD_FAILURE() << "DestroyVideoSendStream called with unknown parameter.";
@@ -526,8 +546,8 @@ webrtc::VideoReceiveStream* FakeCall::CreateVideoReceiveStream(
 
 void FakeCall::DestroyVideoReceiveStream(
     webrtc::VideoReceiveStream* receive_stream) {
-  auto it =
+  auto it = std::find(video_receive_streams_.begin(),
-      std::find(video_receive_streams_.begin(), video_receive_streams_.end(),
+                      video_receive_streams_.end(),
                       static_cast<FakeVideoReceiveStream*>(receive_stream));
   if (it == video_receive_streams_.end()) {
     ADD_FAILURE() << "DestroyVideoReceiveStream called with unknown parameter.";

media/engine/fakewebrtccall.h

@@ -134,6 +134,7 @@ class FakeVideoSendStream final
   bool IsSending() const;
   bool GetVp8Settings(webrtc::VideoCodecVP8* settings) const;
   bool GetVp9Settings(webrtc::VideoCodecVP9* settings) const;
+  bool GetH264Settings(webrtc::VideoCodecH264* settings) const;
 
   int GetNumberOfSwappedFrames() const;
   int GetLastWidth() const;
@@ -179,10 +180,11 @@ class FakeVideoSendStream final
   rtc::VideoSinkWants sink_wants_;
 
   bool codec_settings_set_;
-  union VpxSettings {
+  union CodecSpecificSettings {
     webrtc::VideoCodecVP8 vp8;
     webrtc::VideoCodecVP9 vp9;
-  } vpx_settings_;
+    webrtc::VideoCodecH264 h264;
+  } codec_specific_settings_;
   bool resolution_scaling_enabled_;
   bool framerate_scaling_enabled_;
   rtc::VideoSourceInterface<webrtc::VideoFrame>* source_;

media/engine/simulcast.cc

@@ -15,7 +15,7 @@
 #include "media/base/streamparams.h"
 #include "media/engine/constants.h"
 #include "media/engine/simulcast.h"
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "rtc_base/arraysize.h"
 #include "rtc_base/logging.h"
 #include "system_wrappers/include/field_trial.h"
@@ -52,11 +52,17 @@ struct SimulcastFormat {
 // These tables describe from which resolution we can use how many
 // simulcast layers at what bitrates (maximum, target, and minimum).
 // Important!! Keep this table from high resolution to low resolution.
+// clang-format off
 const SimulcastFormat kSimulcastFormats[] = {
-    {1920, 1080, 3, 5000, 4000, 800}, {1280, 720, 3, 2500, 2500, 600},
+  {1920, 1080, 3, 5000, 4000, 800},
-    {960, 540, 3, 900, 900, 450},     {640, 360, 2, 700, 500, 150},
+  {1280, 720, 3,  2500, 2500, 600},
-    {480, 270, 2, 450, 350, 150},     {320, 180, 1, 200, 150, 30},
+  {960, 540, 3, 900, 900, 450},
-    {0, 0, 1, 200, 150, 30}};
+  {640, 360, 2, 700, 500, 150},
+  {480, 270, 2, 450, 350, 150},
+  {320, 180, 1, 200, 150, 30},
+  {0, 0, 1, 200, 150, 30}
+};
+// clang-format on
 
 const int kMaxScreenshareSimulcastLayers = 2;
 
@@ -173,21 +179,25 @@ int GetTotalMaxBitrateBps(const std::vector<webrtc::VideoStream>& layers) {
   return total_max_bitrate_bps;
 }
 
-std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_layers,
+std::vector<webrtc::VideoStream> GetSimulcastConfig(
+    size_t max_layers,
     int width,
     int height,
     int /*max_bitrate_bps*/,
     double bitrate_priority,
     int max_qp,
     int max_framerate,
-                                                    bool is_screenshare) {
+    bool is_screenshare,
+    bool temporal_layers_supported) {
   if (is_screenshare) {
     return GetScreenshareLayers(max_layers, width, height, bitrate_priority,
                                 max_qp, max_framerate,
-                                ScreenshareSimulcastFieldTrialEnabled());
+                                ScreenshareSimulcastFieldTrialEnabled(),
+                                temporal_layers_supported);
   } else {
     return GetNormalSimulcastLayers(max_layers, width, height, bitrate_priority,
-                                    max_qp, max_framerate);
+                                    max_qp, max_framerate,
+                                    temporal_layers_supported);
   }
 }
 
@@ -197,7 +207,8 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
     int height,
     double bitrate_priority,
     int max_qp,
-    int max_framerate) {
+    int max_framerate,
+    bool temporal_layers_supported) {
   // TODO(bugs.webrtc.org/8785): Currently if the resolution isn't large enough
   // (defined in kSimulcastFormats) we scale down the number of simulcast
   // layers. Consider changing this so that the application can have more
@@ -226,7 +237,9 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
     layers[s].height = height;
     // TODO(pbos): Fill actual temporal-layer bitrate thresholds.
     layers[s].max_qp = max_qp;
-    layers[s].num_temporal_layers = DefaultNumberOfTemporalLayers(s);
+    layers[s].num_temporal_layers =
+        temporal_layers_supported ? DefaultNumberOfTemporalLayers(s)
+                                  : 0;
     layers[s].max_bitrate_bps = FindSimulcastMaxBitrateBps(width, height);
     layers[s].target_bitrate_bps = FindSimulcastTargetBitrateBps(width, height);
     int num_temporal_layers = DefaultNumberOfTemporalLayers(s);
@@ -237,8 +250,8 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
       // with the default 3 simulcast streams. Otherwise we risk a higher
       // threshold for receiving a feed at all.
       const float rate_factor =
-          webrtc::kVp8LayerRateAlloction[3][0] /
+          webrtc::kLayerRateAllocation[3][0] /
-          webrtc::kVp8LayerRateAlloction[num_temporal_layers][0];
+          webrtc::kLayerRateAllocation[num_temporal_layers][0];
       layers[s].max_bitrate_bps =
           static_cast<int>(layers[s].max_bitrate_bps * rate_factor);
       layers[s].target_bitrate_bps =
@@ -270,7 +283,8 @@ std::vector<webrtc::VideoStream> GetScreenshareLayers(
     double bitrate_priority,
     int max_qp,
     int max_framerate,
-    bool screenshare_simulcast_enabled) {
+    bool screenshare_simulcast_enabled,
+    bool temporal_layers_supported) {
   auto max_screenshare_layers =
       screenshare_simulcast_enabled ? kMaxScreenshareSimulcastLayers : 1;
   size_t num_simulcast_layers =
@@ -287,7 +301,7 @@ std::vector<webrtc::VideoStream> GetScreenshareLayers(
   layers[0].min_bitrate_bps = kMinVideoBitrateBps;
   layers[0].target_bitrate_bps = kScreenshareDefaultTl0BitrateKbps * 1000;
   layers[0].max_bitrate_bps = kScreenshareDefaultTl1BitrateKbps * 1000;
-  layers[0].num_temporal_layers = 2;
+  layers[0].num_temporal_layers = temporal_layers_supported ? 2 : 0;
 
   // With simulcast enabled, add another spatial layer. This one will have a
   // more normal layout, with the regular 3 temporal layer pattern and no fps

media/engine/simulcast.h

@@ -31,14 +31,16 @@ void BoostMaxSimulcastLayer(int max_bitrate_bps,
 
 // Gets simulcast settings.
 // TODO(asapersson): Remove max_bitrate_bps.
-std::vector<webrtc::VideoStream> GetSimulcastConfig(size_t max_layers,
+std::vector<webrtc::VideoStream> GetSimulcastConfig(
+    size_t max_layers,
     int width,
     int height,
     int /*max_bitrate_bps*/,
     double bitrate_priority,
     int max_qp,
     int max_framerate,
-                                                    bool is_screenshare);
+    bool is_screenshare,
+    bool temporal_layers_supported = true);
 
 // Gets the simulcast config layers for a non-screensharing case.
 std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
@@ -47,7 +49,8 @@ std::vector<webrtc::VideoStream> GetNormalSimulcastLayers(
     int height,
     double bitrate_priority,
     int max_qp,
-    int max_framerate);
+    int max_framerate,
+    bool temporal_layers_supported = true);
 
 // Gets simulcast config layers for screenshare settings.
 std::vector<webrtc::VideoStream> GetScreenshareLayers(
@@ -57,7 +60,8 @@ std::vector<webrtc::VideoStream> GetScreenshareLayers(
     double bitrate_priority,
     int max_qp,
     int max_framerate,
-    bool screenshare_simulcast_enabled);
+    bool screenshare_simulcast_enabled,
+    bool temporal_layers_supported = true);
 
 bool ScreenshareSimulcastFieldTrialEnabled();
 

media/engine/simulcast_encoder_adapter.cc

@@ -16,8 +16,7 @@
 #include "api/video/video_bitrate_allocation.h"
 #include "api/video_codecs/video_encoder_factory.h"
 #include "media/engine/scopedvideoencoder.h"
-#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
 #include "rtc_base/checks.h"
 #include "system_wrappers/include/clock.h"
 #include "third_party/libyuv/include/libyuv/scale.h"
@@ -76,7 +75,8 @@ int VerifyCodec(const webrtc::VideoCodec* inst) {
   if (inst->width <= 1 || inst->height <= 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
-  if (inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) {
+  if (inst->codecType == webrtc::kVideoCodecVP8 &&
+      inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   return WEBRTC_VIDEO_CODEC_OK;
@@ -219,7 +219,8 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
       encoder = std::move(stored_encoders_.top());
       stored_encoders_.pop();
     } else {
-      encoder = factory_->CreateVideoEncoder(video_format_);
+      encoder = factory_->CreateVideoEncoder(SdpVideoFormat(
+          codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264"));
     }
 
     ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size);
@@ -435,8 +436,11 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
     const RTPFragmentationHeader* fragmentation) {
   CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
   stream_codec_specific.codec_name = implementation_name_.c_str();
-  CodecSpecificInfoVP8* vp8Info = &(stream_codec_specific.codecSpecific.VP8);
+  if (stream_codec_specific.codecType == webrtc::kVideoCodecVP8) {
-  vp8Info->simulcastIdx = stream_idx;
+    stream_codec_specific.codecSpecific.VP8.simulcastIdx = stream_idx;
+  } else if (stream_codec_specific.codecType == webrtc::kVideoCodecH264) {
+    stream_codec_specific.codecSpecific.H264.simulcast_idx = stream_idx;
+  }
 
   return encoded_complete_callback_->OnEncodedImage(
       encodedImage, &stream_codec_specific, fragmentation);
@@ -451,8 +455,6 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
   *stream_codec = inst;
 
   // Stream specific settings.
-  stream_codec->VP8()->numberOfTemporalLayers =
-      inst.simulcastStream[stream_index].numberOfTemporalLayers;
   stream_codec->numberOfSimulcastStreams = 0;
   stream_codec->width = inst.simulcastStream[stream_index].width;
   stream_codec->height = inst.simulcastStream[stream_index].height;
@@ -465,6 +467,9 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
     // Settings for lowest spatial resolutions.
     stream_codec->qpMax = kLowestResMaxQp;
   }
+  if (inst.codecType == webrtc::kVideoCodecVP8) {
+    stream_codec->VP8()->numberOfTemporalLayers =
+        inst.simulcastStream[stream_index].numberOfTemporalLayers;
     if (!highest_resolution_stream) {
       // For resolutions below CIF, set the codec |complexity| parameter to
       // kComplexityHigher, which maps to cpu_used = -4.
@@ -476,6 +481,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
       // Turn off denoising for all streams but the highest resolution.
       stream_codec->VP8()->denoisingOn = false;
     }
+  }
   // TODO(ronghuawu): what to do with targetBitrate.
 
   stream_codec->startBitrate = start_bitrate_kbps;

media/engine/simulcast_encoder_adapter.h

@@ -19,7 +19,7 @@
 #include <vector>
 
 #include "media/engine/webrtcvideoencoderfactory.h"
-#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/include/video_codec_interface.h"
 #include "rtc_base/atomicops.h"
 #include "rtc_base/sequenced_task_checker.h"
 
@@ -32,7 +32,7 @@ class VideoEncoderFactory;
 // webrtc::VideoEncoder instances with the given VideoEncoderFactory.
 // The object is created and destroyed on the worker thread, but all public
 // interfaces should be called from the encoder task queue.
-class SimulcastEncoderAdapter : public VP8Encoder {
+class SimulcastEncoderAdapter : public VideoEncoder {
  public:
   explicit SimulcastEncoderAdapter(VideoEncoderFactory* factory,
                                    const SdpVideoFormat& format);

media/engine/simulcast_encoder_adapter_unittest.cc

@@ -19,8 +19,9 @@
 #include "common_video/include/video_frame_buffer.h"
 #include "media/engine/internalencoderfactory.h"
 #include "media/engine/simulcast_encoder_adapter.h"
-#include "modules/video_coding/codecs/vp8/simulcast_test_fixture_impl.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/simulcast_test_fixture_impl.h"
 #include "rtc_base/ptr_util.h"
 #include "test/function_video_decoder_factory.h"
 #include "test/function_video_encoder_factory.h"
@@ -51,7 +52,8 @@ std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture(
       rtc::MakeUnique<FunctionVideoDecoderFactory>(
           []() { return VP8Decoder::Create(); });
   return CreateSimulcastTestFixture(std::move(encoder_factory),
-                                    std::move(decoder_factory));
+                                    std::move(decoder_factory),
+                                    SdpVideoFormat(cricket::kVp8CodecName));
 }
 
 }  // namespace
@@ -218,6 +220,7 @@ class MockVideoEncoder : public VideoEncoder {
     image._encodedHeight = height;
     CodecSpecificInfo codec_specific_info;
     memset(&codec_specific_info, 0, sizeof(codec_specific_info));
+    codec_specific_info.codecType = webrtc::kVideoCodecVP8;
     callback_->OnEncodedImage(image, &codec_specific_info, nullptr);
   }
 
@@ -295,7 +298,7 @@ class TestSimulcastEncoderAdapterFakeHelper {
 
   // Can only be called once as the SimulcastEncoderAdapter will take the
   // ownership of |factory_|.
-  VP8Encoder* CreateMockEncoderAdapter() {
+  VideoEncoder* CreateMockEncoderAdapter() {
     return new SimulcastEncoderAdapter(factory_.get(), SdpVideoFormat("VP8"));
   }
 
@@ -357,7 +360,8 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
 
   void SetupCodec() {
     SimulcastTestFixtureImpl::DefaultSettings(
-        &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+        &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+        kVideoCodecVP8);
     rate_allocator_.reset(new SimulcastRateAllocator(codec_));
     EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
     adapter_->RegisterEncodeCompleteCallback(this);
@@ -432,7 +436,7 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
 
  protected:
   std::unique_ptr<TestSimulcastEncoderAdapterFakeHelper> helper_;
-  std::unique_ptr<VP8Encoder> adapter_;
+  std::unique_ptr<VideoEncoder> adapter_;
   VideoCodec codec_;
   int last_encoded_image_width_;
   int last_encoded_image_height_;
@@ -506,7 +510,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) {
 TEST_F(TestSimulcastEncoderAdapterFake, ReusesEncodersInOrder) {
   // Set up common settings for three streams.
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   rate_allocator_.reset(new SimulcastRateAllocator(codec_));
   adapter_->RegisterEncodeCompleteCallback(this);
 
@@ -704,7 +709,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) {
 
 TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) {
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   codec_.numberOfSimulcastStreams = 1;
   EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
   adapter_->RegisterEncodeCompleteCallback(this);
@@ -717,7 +723,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) {
 
 TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) {
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   codec_.minBitrate = 50;
   codec_.numberOfSimulcastStreams = 1;
   EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
@@ -746,7 +753,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) {
 TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
   EXPECT_STREQ("SimulcastEncoderAdapter", adapter_->ImplementationName());
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   std::vector<const char*> encoder_names;
   encoder_names.push_back("codec1");
   encoder_names.push_back("codec2");
@@ -768,7 +776,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
 TEST_F(TestSimulcastEncoderAdapterFake,
        SupportsNativeHandleForMultipleStreams) {
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   codec_.numberOfSimulcastStreams = 3;
   EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
   adapter_->RegisterEncodeCompleteCallback(this);
@@ -805,7 +814,8 @@ class FakeNativeBuffer : public VideoFrameBuffer {
 TEST_F(TestSimulcastEncoderAdapterFake,
        NativeHandleForwardingForMultipleStreams) {
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   codec_.numberOfSimulcastStreams = 3;
   // High start bitrate, so all streams are enabled.
   codec_.startBitrate = 3000;
@@ -829,7 +839,8 @@ TEST_F(TestSimulcastEncoderAdapterFake,
 
 TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) {
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   codec_.numberOfSimulcastStreams = 3;
   EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
   adapter_->RegisterEncodeCompleteCallback(this);
@@ -850,7 +861,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) {
 
 TEST_F(TestSimulcastEncoderAdapterFake, TestInitFailureCleansUpEncoders) {
   SimulcastTestFixtureImpl::DefaultSettings(
-      &codec_, static_cast<const int*>(kTestTemporalLayerProfile));
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
   codec_.numberOfSimulcastStreams = 3;
   helper_->factory()->set_init_encode_return_value(
       WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE);

media/engine/webrtcvideoengine.cc

@@ -318,7 +318,9 @@ static bool ValidateStreamParams(const StreamParams& sp) {
 
 // Returns true if the given codec is disallowed from doing simulcast.
 bool IsCodecBlacklistedForSimulcast(const std::string& codec_name) {
-  return CodecNamesEq(codec_name, kH264CodecName) ||
+  return webrtc::field_trial::IsEnabled("WebRTC-H264Simulcast")
+    ? CodecNamesEq(codec_name, kVp9CodecName)
+    : CodecNamesEq(codec_name, kH264CodecName) ||
       CodecNamesEq(codec_name, kVp9CodecName);
 }
 
@@ -2713,11 +2715,14 @@ std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
   std::vector<webrtc::VideoStream> layers;
 
   if (encoder_config.number_of_streams > 1 ||
-      (CodecNamesEq(codec_name_, kVp8CodecName) && is_screenshare_ &&
+     ((CodecNamesEq(codec_name_, kVp8CodecName) ||
-       screenshare_config_explicitly_enabled_)) {
+        CodecNamesEq(codec_name_, kH264CodecName)) &&
+       is_screenshare_ && screenshare_config_explicitly_enabled_)) {
+    bool temporal_layers_supported = CodecNamesEq(codec_name_, kVp8CodecName);
     layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height,
                                 0 /*not used*/, encoder_config.bitrate_priority,
-                                max_qp_, max_framerate_, is_screenshare_);
+                                max_qp_, max_framerate_, is_screenshare_,
+                                temporal_layers_supported);
     // Update the active simulcast layers and configured bitrates.
     bool is_highest_layer_max_bitrate_configured = false;
     for (size_t i = 0; i < layers.size(); ++i) {

media/engine/webrtcvideoengine_unittest.cc

@@ -770,7 +770,9 @@ TEST_F(WebRtcVideoEngineTest,
   ASSERT_EQ(0u, encoder_factory_->encoders().size());
 }
 
-TEST_F(WebRtcVideoEngineTest, SimulcastDisabledForH264) {
+TEST_F(WebRtcVideoEngineTest, SimulcastEnabledForH264BehindFieldTrial) {
+  webrtc::test::ScopedFieldTrials override_field_trials_(
+      "WebRTC-H264Simulcast/Enabled/");
   encoder_factory_->AddSupportedVideoCodecType("H264");
 
   std::unique_ptr<VideoMediaChannel> channel(
@@ -796,7 +798,7 @@ TEST_F(WebRtcVideoEngineTest, SimulcastDisabledForH264) {
   FakeWebRtcVideoEncoder* encoder = encoder_factory_->encoders()[0];
   ASSERT_TRUE(encoder_factory_->encoders()[0]->WaitForInitEncode());
   EXPECT_EQ(webrtc::kVideoCodecH264, encoder->GetCodecSettings().codecType);
-  EXPECT_EQ(1u, encoder->GetCodecSettings().numberOfSimulcastStreams);
+  EXPECT_LT(1u, encoder->GetCodecSettings().numberOfSimulcastStreams);
   EXPECT_TRUE(channel->SetVideoSend(ssrcs[0], nullptr, nullptr));
 }
 
@@ -6184,7 +6186,7 @@ class WebRtcVideoChannelSimulcastTest : public testing::Test {
       expected_streams = GetSimulcastConfig(
           num_configured_streams, capture_width, capture_height, 0,
           webrtc::kDefaultBitratePriority, kDefaultQpMax,
-          kDefaultVideoMaxFramerate, screenshare);
+          kDefaultVideoMaxFramerate, screenshare, true);
       if (screenshare) {
         for (const webrtc::VideoStream& stream : expected_streams) {
           // Never scale screen content.

modules/rtp_rtcp/source/rtp_format_h264.cc

@@ -456,6 +456,7 @@ bool RtpDepacketizerH264::ProcessStapAOrSingleNalu(
   parsed_payload->type.Video.width = 0;
   parsed_payload->type.Video.height = 0;
   parsed_payload->type.Video.codec = kVideoCodecH264;
+  parsed_payload->type.Video.simulcastIdx = 0;
   parsed_payload->type.Video.is_first_packet_in_frame = true;
   RTPVideoHeaderH264* h264_header =
       &parsed_payload->type.Video.codecHeader.H264;
@@ -674,6 +675,7 @@ bool RtpDepacketizerH264::ParseFuaNalu(
   parsed_payload->type.Video.width = 0;
   parsed_payload->type.Video.height = 0;
   parsed_payload->type.Video.codec = kVideoCodecH264;
+  parsed_payload->type.Video.simulcastIdx = 0;
   parsed_payload->type.Video.is_first_packet_in_frame = first_fragment;
   RTPVideoHeaderH264* h264 = &parsed_payload->type.Video.codecHeader.H264;
   h264->packetization_type = kH264FuA;

modules/video_coding/BUILD.gn

@@ -152,7 +152,6 @@ rtc_static_library("video_coding") {
     ":packet",
     ":video_codec_interface",
     ":video_coding_utility",
-    ":webrtc_vp8_helpers",
     ":webrtc_vp9_helpers",
     "..:module_api",
     "..:module_api_public",
@@ -237,6 +236,10 @@ rtc_source_set("video_coding_utility") {
     "utility/moving_average.h",
     "utility/quality_scaler.cc",
     "utility/quality_scaler.h",
+    "utility/simulcast_rate_allocator.cc",
+    "utility/simulcast_rate_allocator.h",
+    "utility/simulcast_utility.cc",
+    "utility/simulcast_utility.h",
     "utility/vp8_header_parser.cc",
     "utility/vp8_header_parser.h",
     "utility/vp9_uncompressed_header_parser.cc",
@@ -263,6 +266,8 @@ rtc_source_set("video_coding_utility") {
     "../../rtc_base:sequenced_task_checker",
     "../../rtc_base/experiments:quality_scaling_experiment",
     "../../system_wrappers",
+    "../../system_wrappers:field_trial_api",
+    "../../system_wrappers:metrics_api",
     "../rtp_rtcp:rtp_rtcp_format",
     "//third_party/abseil-cpp/absl/types:optional",
   ]
@@ -291,6 +296,7 @@ rtc_static_library("webrtc_h264") {
     "../../rtc_base:checks",
     "../../rtc_base:rtc_base",
     "../../system_wrappers:metrics_api",
+    "//third_party/libyuv",
   ]
 
   if (rtc_use_h264) {
@@ -377,18 +383,20 @@ rtc_static_library("webrtc_multiplex") {
   ]
 }
 
-# This target includes VP8 files that may be used for any VP8 codec, internal SW or external HW.
+# This target includes the internal SW codec.
-rtc_static_library("webrtc_vp8_helpers") {
+rtc_static_library("webrtc_vp8") {
   visibility = [ "*" ]
+  poisonous = [ "software_video_codecs" ]
   sources = [
     "codecs/vp8/default_temporal_layers.cc",
     "codecs/vp8/default_temporal_layers.h",
     "codecs/vp8/include/vp8.h",
-    "codecs/vp8/include/vp8_common_types.h",
+    "codecs/vp8/libvpx_vp8_decoder.cc",
+    "codecs/vp8/libvpx_vp8_decoder.h",
+    "codecs/vp8/libvpx_vp8_encoder.cc",
+    "codecs/vp8/libvpx_vp8_encoder.h",
     "codecs/vp8/screenshare_layers.cc",
     "codecs/vp8/screenshare_layers.h",
-    "codecs/vp8/simulcast_rate_allocator.cc",
-    "codecs/vp8/simulcast_rate_allocator.h",
     "codecs/vp8/temporal_layers.cc",
     "codecs/vp8/temporal_layers.h",
   ]
@@ -416,45 +424,6 @@ rtc_static_library("webrtc_vp8_helpers") {
     "//third_party/abseil-cpp/absl/types:optional",
     "//third_party/libyuv",
   ]
-}
-
-# This target includes the internal SW codec.
-rtc_static_library("webrtc_vp8") {
-  visibility = [ "*" ]
-  poisonous = [ "software_video_codecs" ]
-  sources = [
-    "codecs/vp8/include/vp8.h",
-    "codecs/vp8/include/vp8_common_types.h",
-    "codecs/vp8/libvpx_vp8_decoder.cc",
-    "codecs/vp8/libvpx_vp8_decoder.h",
-    "codecs/vp8/libvpx_vp8_encoder.cc",
-    "codecs/vp8/libvpx_vp8_encoder.h",
-  ]
-
-  if (!build_with_chromium && is_clang) {
-    # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163).
-    suppressed_configs += [ "//build/config/clang:find_bad_constructs" ]
-  }
-
-  deps = [
-    ":video_codec_interface",
-    ":video_coding_utility",
-    ":webrtc_vp8_helpers",
-    "..:module_api",
-    "../..:webrtc_common",
-    "../../:typedefs",
-    "../../api/video:video_frame",
-    "../../api/video_codecs:video_codecs_api",
-    "../../common_video",
-    "../../rtc_base:checks",
-    "../../rtc_base:rtc_base_approved",
-    "../../rtc_base:rtc_numerics",
-    "../../system_wrappers",
-    "../../system_wrappers:field_trial_api",
-    "../../system_wrappers:metrics_api",
-    "//third_party/abseil-cpp/absl/types:optional",
-    "//third_party/libyuv",
-  ]
   if (rtc_build_libvpx) {
     deps += [ rtc_libvpx_dir ]
   }
@@ -574,8 +543,8 @@ if (rtc_include_tests) {
   rtc_source_set("simulcast_test_fixture_impl") {
     testonly = true
     sources = [
-      "codecs/vp8/simulcast_test_fixture_impl.cc",
+      "utility/simulcast_test_fixture_impl.cc",
-      "codecs/vp8/simulcast_test_fixture_impl.h",
+      "utility/simulcast_test_fixture_impl.h",
     ]
 
     if (!build_with_chromium && is_clang) {
@@ -587,7 +556,7 @@ if (rtc_include_tests) {
       ":mock_headers",
       ":video_codec_interface",
       ":video_coding",
-      ":webrtc_vp8_helpers",
+      ":video_coding_utility",
       "../../:webrtc_common",
       "../../api:simulcast_test_fixture_api",
       "../../api/video:video_frame",
@@ -618,7 +587,6 @@ if (rtc_include_tests) {
       ":video_codec_interface",
       ":video_coding",
       ":video_coding_utility",
-      ":webrtc_vp8_helpers",
       ":webrtc_vp9_helpers",
       "../..:webrtc_common",
       "../../:typedefs",
@@ -738,7 +706,7 @@ if (rtc_include_tests) {
       ":videocodec_test_impl",
       ":webrtc_h264",
       ":webrtc_multiplex",
-      ":webrtc_vp8_helpers",
+      ":webrtc_vp8",
       ":webrtc_vp9",
       ":webrtc_vp9_helpers",
       "../..:webrtc_common",
@@ -828,7 +796,10 @@ if (rtc_include_tests) {
       "video_sender_unittest.cc",
     ]
     if (rtc_use_h264) {
-      sources += [ "codecs/h264/h264_encoder_impl_unittest.cc" ]
+      sources += [
+        "codecs/h264/h264_encoder_impl_unittest.cc",
+        "codecs/h264/h264_simulcast_unittest.cc",
+      ]
     }
 
     deps = [
@@ -837,6 +808,7 @@ if (rtc_include_tests) {
       ":mock_headers",
       ":nack_module",
       ":packet",
+      ":simulcast_test_fixture_impl",
       ":video_codec_interface",
       ":video_codecs_test_framework",
       ":video_coding",
@@ -844,7 +816,6 @@ if (rtc_include_tests) {
       ":videocodec_test_impl",
       ":webrtc_h264",
       ":webrtc_vp8",
-      ":webrtc_vp8_helpers",
       ":webrtc_vp9",
       ":webrtc_vp9_helpers",
       "..:module_api",

modules/video_coding/codecs/h264/h264_encoder_impl.cc

@@ -20,10 +20,14 @@
 #include "third_party/openh264/src/codec/api/svc/codec_ver.h"
 
 #include "common_video/libyuv/include/webrtc_libyuv.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_utility.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/timeutils.h"
 #include "system_wrappers/include/metrics.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
 
 namespace webrtc {
 
@@ -157,16 +161,7 @@ static void RtpFragmentize(EncodedImage* encoded_image,
 }
 
 H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
-    : openh264_encoder_(nullptr),
+    : packetization_mode_(H264PacketizationMode::SingleNalUnit),
-      width_(0),
-      height_(0),
-      max_frame_rate_(0.0f),
-      target_bps_(0),
-      max_bps_(0),
-      mode_(VideoCodecMode::kRealtimeVideo),
-      frame_dropping_on_(false),
-      key_frame_interval_(0),
-      packetization_mode_(H264PacketizationMode::SingleNalUnit),
       max_payload_size_(0),
       number_of_cores_(0),
       encoded_image_callback_(nullptr),
@@ -179,25 +174,30 @@ H264EncoderImpl::H264EncoderImpl(const cricket::VideoCodec& codec)
       packetization_mode_string == "1") {
     packetization_mode_ = H264PacketizationMode::NonInterleaved;
   }
+  downscaled_buffers_.reserve(kMaxSimulcastStreams - 1);
+  encoded_images_.reserve(kMaxSimulcastStreams);
+  encoded_image_buffers_.reserve(kMaxSimulcastStreams);
+  encoders_.reserve(kMaxSimulcastStreams);
+  configurations_.reserve(kMaxSimulcastStreams);
 }
 
 H264EncoderImpl::~H264EncoderImpl() {
   Release();
 }
 
-int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
+int32_t H264EncoderImpl::InitEncode(const VideoCodec* inst,
                                     int32_t number_of_cores,
                                     size_t max_payload_size) {
   ReportInit();
-  if (!codec_settings || codec_settings->codecType != kVideoCodecH264) {
+  if (!inst || inst->codecType != kVideoCodecH264) {
     ReportError();
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
-  if (codec_settings->maxFramerate == 0) {
+  if (inst->maxFramerate == 0) {
     ReportError();
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
-  if (codec_settings->width < 1 || codec_settings->height < 1) {
+  if (inst->width < 1 || inst->height < 1) {
     ReportError();
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
@@ -207,44 +207,91 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
     ReportError();
     return release_ret;
   }
-  RTC_DCHECK(!openh264_encoder_);
 
+  int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
+  bool doing_simulcast = (number_of_streams > 1);
+
+  if (doing_simulcast && (!SimulcastUtility::ValidSimulcastResolutions(
+                              *inst, number_of_streams) ||
+                          !SimulcastUtility::ValidSimulcastTemporalLayers(
+                              *inst, number_of_streams))) {
+    return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
+  }
+  downscaled_buffers_.resize(number_of_streams - 1);
+  encoded_images_.resize(number_of_streams);
+  encoded_image_buffers_.resize(number_of_streams);
+  encoders_.resize(number_of_streams);
+  pictures_.resize(number_of_streams);
+  configurations_.resize(number_of_streams);
+
+  number_of_cores_ = number_of_cores;
+  max_payload_size_ = max_payload_size;
+  codec_ = *inst;
+
+  // Code expects simulcastStream resolutions to be correct, make sure they are
+  // filled even when there are no simulcast layers.
+  if (codec_.numberOfSimulcastStreams == 0) {
+    codec_.simulcastStream[0].width = codec_.width;
+    codec_.simulcastStream[0].height = codec_.height;
+  }
+
+  for (int i = 0, idx = number_of_streams - 1; i < number_of_streams;
+       ++i, --idx) {
+    // Temporal layers still not supported.
+    if (inst->simulcastStream[i].numberOfTemporalLayers > 1) {
+      Release();
+      return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
+    }
+    ISVCEncoder* openh264_encoder;
     // Create encoder.
-  if (WelsCreateSVCEncoder(&openh264_encoder_) != 0) {
+    if (WelsCreateSVCEncoder(&openh264_encoder) != 0) {
       // Failed to create encoder.
       RTC_LOG(LS_ERROR) << "Failed to create OpenH264 encoder";
-    RTC_DCHECK(!openh264_encoder_);
+      RTC_DCHECK(!openh264_encoder);
+      Release();
       ReportError();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
-  RTC_DCHECK(openh264_encoder_);
+    RTC_DCHECK(openh264_encoder);
     if (kOpenH264EncoderDetailedLogging) {
       int trace_level = WELS_LOG_DETAIL;
-    openh264_encoder_->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
+      openh264_encoder->SetOption(ENCODER_OPTION_TRACE_LEVEL, &trace_level);
     }
     // else WELS_LOG_DEFAULT is used by default.
 
-  number_of_cores_ = number_of_cores;
+    // Store h264 encoder.
+    encoders_[i] = openh264_encoder;
+
     // Set internal settings from codec_settings
-  width_ = codec_settings->width;
+    configurations_[i].simulcast_idx = idx;
-  height_ = codec_settings->height;
+    configurations_[i].sending = false;
-  max_frame_rate_ = static_cast<float>(codec_settings->maxFramerate);
+    configurations_[i].width = codec_.simulcastStream[idx].width;
-  mode_ = codec_settings->mode;
+    configurations_[i].height = codec_.simulcastStream[idx].height;
-  frame_dropping_on_ = codec_settings->H264().frameDroppingOn;
+    configurations_[i].max_frame_rate = static_cast<float>(codec_.maxFramerate);
-  key_frame_interval_ = codec_settings->H264().keyFrameInterval;
+    configurations_[i].frame_dropping_on = codec_.H264()->frameDroppingOn;
-  max_payload_size_ = max_payload_size;
+    configurations_[i].key_frame_interval = codec_.H264()->keyFrameInterval;
+
+    // Create downscaled image buffers.
+    if (i > 0) {
+      downscaled_buffers_[i - 1] = I420Buffer::Create(
+          configurations_[i].width, configurations_[i].height,
+          configurations_[i].width, configurations_[i].width / 2,
+          configurations_[i].width / 2);
+    }
 
     // Codec_settings uses kbits/second; encoder uses bits/second.
-  max_bps_ = codec_settings->maxBitrate * 1000;
+    configurations_[i].max_bps = codec_.maxBitrate * 1000;
-  if (codec_settings->targetBitrate == 0)
+    if (codec_.targetBitrate == 0) {
-    target_bps_ = codec_settings->startBitrate * 1000;
+      configurations_[i].target_bps = codec_.startBitrate * 1000;
-  else
+    } else {
-    target_bps_ = codec_settings->targetBitrate * 1000;
+      configurations_[i].target_bps = codec_.targetBitrate * 1000;
+    }
 
-  SEncParamExt encoder_params = CreateEncoderParams();
+    // Create encoder parameters based on the layer configuration.
+    SEncParamExt encoder_params = CreateEncoderParams(i);
 
     // Initialize.
-  if (openh264_encoder_->InitializeExt(&encoder_params) != 0) {
+    if (openh264_encoder->InitializeExt(&encoder_params) != 0) {
       RTC_LOG(LS_ERROR) << "Failed to initialize OpenH264 encoder";
       Release();
       ReportError();
@@ -252,28 +299,42 @@ int32_t H264EncoderImpl::InitEncode(const VideoCodec* codec_settings,
     }
     // TODO(pbos): Base init params on these values before submitting.
     int video_format = EVideoFormatType::videoFormatI420;
-  openh264_encoder_->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format);
+    openh264_encoder->SetOption(ENCODER_OPTION_DATAFORMAT, &video_format);
 
     // Initialize encoded image. Default buffer size: size of unencoded data.
-  encoded_image_._size = CalcBufferSize(VideoType::kI420, codec_settings->width,
+    encoded_images_[i]._size =
-                                        codec_settings->height);
+        CalcBufferSize(VideoType::kI420, codec_.simulcastStream[idx].width,
-  encoded_image_._buffer = new uint8_t[encoded_image_._size];
+                       codec_.simulcastStream[idx].height);
-  encoded_image_buffer_.reset(encoded_image_._buffer);
+    encoded_images_[i]._buffer = new uint8_t[encoded_images_[i]._size];
-  encoded_image_._completeFrame = true;
+    encoded_image_buffers_[i].reset(encoded_images_[i]._buffer);
-  encoded_image_._encodedWidth = 0;
+    encoded_images_[i]._completeFrame = true;
-  encoded_image_._encodedHeight = 0;
+    encoded_images_[i]._encodedWidth = codec_.simulcastStream[idx].width;
-  encoded_image_._length = 0;
+    encoded_images_[i]._encodedHeight = codec_.simulcastStream[idx].height;
-  return WEBRTC_VIDEO_CODEC_OK;
+    encoded_images_[i]._length = 0;
+  }
+
+  SimulcastRateAllocator init_allocator(codec_);
+  BitrateAllocation allocation = init_allocator.GetAllocation(
+      codec_.targetBitrate ? codec_.targetBitrate * 1000
+                           : codec_.startBitrate * 1000,
+      codec_.maxFramerate);
+  return SetRateAllocation(allocation, codec_.maxFramerate);
 }
 
 int32_t H264EncoderImpl::Release() {
-  if (openh264_encoder_) {
+  while (!encoders_.empty()) {
-    RTC_CHECK_EQ(0, openh264_encoder_->Uninitialize());
+    ISVCEncoder* openh264_encoder = encoders_.back();
-    WelsDestroySVCEncoder(openh264_encoder_);
+    if (openh264_encoder) {
-    openh264_encoder_ = nullptr;
+      RTC_CHECK_EQ(0, openh264_encoder->Uninitialize());
+      WelsDestroySVCEncoder(openh264_encoder);
     }
-  encoded_image_._buffer = nullptr;
+    encoders_.pop_back();
-  encoded_image_buffer_.reset();
+  }
+  downscaled_buffers_.clear();
+  configurations_.clear();
+  encoded_images_.clear();
+  encoded_image_buffers_.clear();
+  pictures_.clear();
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ -284,27 +345,59 @@ int32_t H264EncoderImpl::RegisterEncodeCompleteCallback(
 }
 
 int32_t H264EncoderImpl::SetRateAllocation(
-    const VideoBitrateAllocation& bitrate_allocation,
+    const BitrateAllocation& bitrate,
-    uint32_t framerate) {
+    uint32_t new_framerate) {
-  if (bitrate_allocation.get_sum_bps() <= 0 || framerate <= 0)
+  if (encoders_.empty())
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+
+  if (new_framerate < 1)
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
 
-  target_bps_ = bitrate_allocation.get_sum_bps();
+  if (bitrate.get_sum_bps() == 0) {
-  max_frame_rate_ = static_cast<float>(framerate);
+    // Encoder paused, turn off all encoding.
+    for (size_t i = 0; i < configurations_.size(); ++i)
+      configurations_[i].SetStreamState(false);
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
 
+  // At this point, bitrate allocation should already match codec settings.
+  if (codec_.maxBitrate > 0)
+    RTC_DCHECK_LE(bitrate.get_sum_kbps(), codec_.maxBitrate);
+  RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.minBitrate);
+  if (codec_.numberOfSimulcastStreams > 0)
+    RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.simulcastStream[0].minBitrate);
+
+  codec_.maxFramerate = new_framerate;
+
+  size_t stream_idx = encoders_.size() - 1;
+  for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
+    // Update layer config.
+    configurations_[i].target_bps = bitrate.GetSpatialLayerSum(stream_idx);
+    configurations_[i].max_frame_rate = static_cast<float>(new_framerate);
+
+    if (configurations_[i].target_bps) {
+      configurations_[i].SetStreamState(true);
+
+      // Update h264 encoder.
       SBitrateInfo target_bitrate;
       memset(&target_bitrate, 0, sizeof(SBitrateInfo));
       target_bitrate.iLayer = SPATIAL_LAYER_ALL,
-  target_bitrate.iBitrate = target_bps_;
+      target_bitrate.iBitrate = configurations_[i].target_bps;
-  openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate);
+      encoders_[i]->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate);
-  openh264_encoder_->SetOption(ENCODER_OPTION_FRAME_RATE, &max_frame_rate_);
+      encoders_[i]->SetOption(ENCODER_OPTION_FRAME_RATE,
+                              &configurations_[i].max_frame_rate);
+    } else {
+      configurations_[i].SetStreamState(false);
+    }
+  }
+
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame,
                                 const CodecSpecificInfo* codec_specific_info,
                                 const std::vector<FrameType>* frame_types) {
-  if (!IsInitialized()) {
+  if (encoders_.empty()) {
     ReportError();
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
@@ -316,83 +409,134 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame,
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
-  bool force_key_frame = false;
+  rtc::scoped_refptr<const I420BufferInterface> frame_buffer =
+      input_frame.video_frame_buffer()->ToI420();
+
+  bool send_key_frame = false;
+  for (size_t i = 0; i < configurations_.size(); ++i) {
+    if (configurations_[i].key_frame_request && configurations_[i].sending) {
+      send_key_frame = true;
+      break;
+    }
+  }
+  if (!send_key_frame && frame_types) {
+    for (size_t i = 0; i < frame_types->size() && i < configurations_.size();
+         ++i) {
+      if ((*frame_types)[i] == kVideoFrameKey && configurations_[i].sending) {
+        send_key_frame = true;
+        break;
+      }
+    }
+  }
+
+  RTC_DCHECK_EQ(configurations_[0].width, frame_buffer->width());
+  RTC_DCHECK_EQ(configurations_[0].height, frame_buffer->height());
+
+  // Encode image for each layer.
+  for (size_t i = 0; i < encoders_.size(); ++i) {
+    // EncodeFrame input.
+    pictures_[i] = {0};
+    pictures_[i].iPicWidth = configurations_[i].width;
+    pictures_[i].iPicHeight = configurations_[i].height;
+    pictures_[i].iColorFormat = EVideoFormatType::videoFormatI420;
+    pictures_[i].uiTimeStamp = input_frame.ntp_time_ms();
+    // Downscale images on second and ongoing layers.
+    if (i == 0) {
+      pictures_[i].iStride[0] = frame_buffer->StrideY();
+      pictures_[i].iStride[1] = frame_buffer->StrideU();
+      pictures_[i].iStride[2] = frame_buffer->StrideV();
+      pictures_[i].pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
+      pictures_[i].pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
+      pictures_[i].pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
+    } else {
+      pictures_[i].iStride[0] = downscaled_buffers_[i - 1]->StrideY();
+      pictures_[i].iStride[1] = downscaled_buffers_[i - 1]->StrideU();
+      pictures_[i].iStride[2] = downscaled_buffers_[i - 1]->StrideV();
+      pictures_[i].pData[0] =
+          const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataY());
+      pictures_[i].pData[1] =
+          const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataU());
+      pictures_[i].pData[2] =
+          const_cast<uint8_t*>(downscaled_buffers_[i - 1]->DataV());
+      // Scale the image down a number of times by downsampling factor.
+      libyuv::I420Scale(pictures_[i - 1].pData[0], pictures_[i - 1].iStride[0],
+                        pictures_[i - 1].pData[1], pictures_[i - 1].iStride[1],
+                        pictures_[i - 1].pData[2], pictures_[i - 1].iStride[2],
+                        configurations_[i - 1].width,
+                        configurations_[i - 1].height, pictures_[i].pData[0],
+                        pictures_[i].iStride[0], pictures_[i].pData[1],
+                        pictures_[i].iStride[1], pictures_[i].pData[2],
+                        pictures_[i].iStride[2], configurations_[i].width,
+                        configurations_[i].height, libyuv::kFilterBilinear);
+    }
+
+    if (!configurations_[i].sending) {
+      continue;
+    }
     if (frame_types != nullptr) {
-    // We only support a single stream.
-    RTC_DCHECK_EQ(frame_types->size(), 1);
       // Skip frame?
-    if ((*frame_types)[0] == kEmptyFrame) {
+      if ((*frame_types)[i] == kEmptyFrame) {
-      return WEBRTC_VIDEO_CODEC_OK;
+        continue;
       }
-    // Force key frame?
-    force_key_frame = (*frame_types)[0] == kVideoFrameKey;
     }
-  if (force_key_frame) {
+    if (send_key_frame) {
       // API doc says ForceIntraFrame(false) does nothing, but calling this
       // function forces a key frame regardless of the |bIDR| argument's value.
       // (If every frame is a key frame we get lag/delays.)
-    openh264_encoder_->ForceIntraFrame(true);
+      encoders_[i]->ForceIntraFrame(true);
+      configurations_[i].key_frame_request = false;
     }
-  rtc::scoped_refptr<const I420BufferInterface> frame_buffer =
-      input_frame.video_frame_buffer()->ToI420();
-  // EncodeFrame input.
-  SSourcePicture picture;
-  memset(&picture, 0, sizeof(SSourcePicture));
-  picture.iPicWidth = frame_buffer->width();
-  picture.iPicHeight = frame_buffer->height();
-  picture.iColorFormat = EVideoFormatType::videoFormatI420;
-  picture.uiTimeStamp = input_frame.ntp_time_ms();
-  picture.iStride[0] = frame_buffer->StrideY();
-  picture.iStride[1] = frame_buffer->StrideU();
-  picture.iStride[2] = frame_buffer->StrideV();
-  picture.pData[0] = const_cast<uint8_t*>(frame_buffer->DataY());
-  picture.pData[1] = const_cast<uint8_t*>(frame_buffer->DataU());
-  picture.pData[2] = const_cast<uint8_t*>(frame_buffer->DataV());
-
     // EncodeFrame output.
     SFrameBSInfo info;
     memset(&info, 0, sizeof(SFrameBSInfo));
 
     // Encode!
-  int enc_ret = openh264_encoder_->EncodeFrame(&picture, &info);
+    int enc_ret = encoders_[i]->EncodeFrame(&pictures_[i], &info);
     if (enc_ret != 0) {
-    RTC_LOG(LS_ERROR) << "OpenH264 frame encoding failed, EncodeFrame returned "
+      RTC_LOG(LS_ERROR)
-                      << enc_ret << ".";
+          << "OpenH264 frame encoding failed, EncodeFrame returned " << enc_ret
+          << ".";
       ReportError();
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
 
-  encoded_image_._encodedWidth = frame_buffer->width();
+    encoded_images_[i]._encodedWidth = configurations_[i].width;
-  encoded_image_._encodedHeight = frame_buffer->height();
+    encoded_images_[i]._encodedHeight = configurations_[i].height;
-  encoded_image_._timeStamp = input_frame.timestamp();
+    encoded_images_[i]._timeStamp = input_frame.timestamp();
-  encoded_image_.ntp_time_ms_ = input_frame.ntp_time_ms();
+    encoded_images_[i].ntp_time_ms_ = input_frame.ntp_time_ms();
-  encoded_image_.capture_time_ms_ = input_frame.render_time_ms();
+    encoded_images_[i].capture_time_ms_ = input_frame.render_time_ms();
-  encoded_image_.rotation_ = input_frame.rotation();
+    encoded_images_[i].rotation_ = input_frame.rotation();
-  encoded_image_.content_type_ = (mode_ == VideoCodecMode::kScreensharing)
+    encoded_images_[i].content_type_ =
+            (codec_.mode == VideoCodecMode::kScreensharing)
             ? VideoContentType::SCREENSHARE
             : VideoContentType::UNSPECIFIED;
-  encoded_image_.timing_.flags = VideoSendTiming::kInvalid;
+    encoded_images_[i].timing_.flags = VideoSendTiming::kInvalid;
-  encoded_image_._frameType = ConvertToVideoFrameType(info.eFrameType);
+    encoded_images_[i]._frameType = ConvertToVideoFrameType(info.eFrameType);
 
-  // Split encoded image up into fragments. This also updates |encoded_image_|.
+    // Split encoded image up into fragments. This also updates
+    // |encoded_image_|.
     RTPFragmentationHeader frag_header;
-  RtpFragmentize(&encoded_image_, &encoded_image_buffer_, *frame_buffer, &info,
+    RtpFragmentize(&encoded_images_[i], &encoded_image_buffers_[i],
-                 &frag_header);
+                   *frame_buffer, &info, &frag_header);
 
     // Encoder can skip frames to save bandwidth in which case
-  // |encoded_image_._length| == 0.
+    // |encoded_images_[i]._length| == 0.
-  if (encoded_image_._length > 0) {
+    if (encoded_images_[i]._length > 0) {
       // Parse QP.
-    h264_bitstream_parser_.ParseBitstream(encoded_image_._buffer,
+      h264_bitstream_parser_.ParseBitstream(encoded_images_[i]._buffer,
-                                          encoded_image_._length);
+                                            encoded_images_[i]._length);
-    h264_bitstream_parser_.GetLastSliceQp(&encoded_image_.qp_);
+      h264_bitstream_parser_.GetLastSliceQp(&encoded_images_[i].qp_);
 
       // Deliver encoded image.
       CodecSpecificInfo codec_specific;
       codec_specific.codecType = kVideoCodecH264;
-    codec_specific.codecSpecific.H264.packetization_mode = packetization_mode_;
+      codec_specific.codecSpecific.H264.packetization_mode =
-    encoded_image_callback_->OnEncodedImage(encoded_image_, &codec_specific,
+          packetization_mode_;
-                                            &frag_header);
+      codec_specific.codecSpecific.H264.simulcast_idx =
+          configurations_[i].simulcast_idx;
+      encoded_image_callback_->OnEncodedImage(encoded_images_[i],
+                                              &codec_specific, &frag_header);
+    }
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
@@ -401,40 +545,35 @@ const char* H264EncoderImpl::ImplementationName() const {
   return "OpenH264";
 }
 
-bool H264EncoderImpl::IsInitialized() const {
-  return openh264_encoder_ != nullptr;
-}
-
 // Initialization parameters.
 // There are two ways to initialize. There is SEncParamBase (cleared with
 // memset(&p, 0, sizeof(SEncParamBase)) used in Initialize, and SEncParamExt
 // which is a superset of SEncParamBase (cleared with GetDefaultParams) used
 // in InitializeExt.
-SEncParamExt H264EncoderImpl::CreateEncoderParams() const {
+SEncParamExt H264EncoderImpl::CreateEncoderParams(size_t i) const {
-  RTC_DCHECK(openh264_encoder_);
   SEncParamExt encoder_params;
-  openh264_encoder_->GetDefaultParams(&encoder_params);
+  encoders_[i]->GetDefaultParams(&encoder_params);
-  if (mode_ == VideoCodecMode::kRealtimeVideo) {
+  if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
     encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
-  } else if (mode_ == VideoCodecMode::kScreensharing) {
+  } else if (codec_.mode == VideoCodecMode::kScreensharing) {
     encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
   } else {
     RTC_NOTREACHED();
   }
-  encoder_params.iPicWidth = width_;
+  encoder_params.iPicWidth = configurations_[i].width;
-  encoder_params.iPicHeight = height_;
+  encoder_params.iPicHeight = configurations_[i].height;
-  encoder_params.iTargetBitrate = target_bps_;
+  encoder_params.iTargetBitrate = configurations_[i].target_bps;
-  encoder_params.iMaxBitrate = max_bps_;
+  encoder_params.iMaxBitrate = configurations_[i].max_bps;
   // Rate Control mode
   encoder_params.iRCMode = RC_BITRATE_MODE;
-  encoder_params.fMaxFrameRate = max_frame_rate_;
+  encoder_params.fMaxFrameRate = configurations_[i].max_frame_rate;
 
   // The following parameters are extension parameters (they're in SEncParamExt,
   // not in SEncParamBase).
-  encoder_params.bEnableFrameSkip = frame_dropping_on_;
+  encoder_params.bEnableFrameSkip = configurations_[i].frame_dropping_on;
   // |uiIntraPeriod|    - multiple of GOP size
   // |keyFrameInterval| - number of frames
-  encoder_params.uiIntraPeriod = key_frame_interval_;
+  encoder_params.uiIntraPeriod = configurations_[i].key_frame_interval;
   encoder_params.uiMaxNalSize = 0;
   // Threading model: use auto.
   //  0: auto (dynamic imp. internal encoder)
@@ -502,4 +641,12 @@ VideoEncoder::ScalingSettings H264EncoderImpl::GetScalingSettings() const {
                                        kHighH264QpThreshold);
 }
 
+void H264EncoderImpl::LayerConfig::SetStreamState(bool send_stream) {
+  if (send_stream && !sending) {
+    // Need a key frame if we have not sent this stream before.
+    key_frame_request = true;
+  }
+  sending = send_stream;
+}
+
 }  // namespace webrtc

modules/video_coding/codecs/h264/h264_encoder_impl.h

@@ -15,6 +15,7 @@
 #include <memory>
 #include <vector>
 
+#include "api/video/i420_buffer.h"
 #include "common_video/h264/h264_bitstream_parser.h"
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/utility/quality_scaler.h"
@@ -26,6 +27,22 @@ class ISVCEncoder;
 namespace webrtc {
 
 class H264EncoderImpl : public H264Encoder {
+ public:
+  struct LayerConfig {
+    int simulcast_idx = 0;
+    int width = -1;
+    int height = -1;
+    bool sending = true;
+    bool key_frame_request = false;
+    float max_frame_rate = 0;
+    uint32_t target_bps = 0;
+    uint32_t max_bps = 0;
+    bool frame_dropping_on = false;
+    int key_frame_interval = 0;
+
+    void SetStreamState(bool send_stream);
+  };
+
  public:
   explicit H264EncoderImpl(const cricket::VideoCodec& codec);
   ~H264EncoderImpl() override;
@@ -66,32 +83,24 @@ class H264EncoderImpl : public H264Encoder {
   }
 
  private:
-  bool IsInitialized() const;
+  SEncParamExt CreateEncoderParams(size_t i) const;
-  SEncParamExt CreateEncoderParams() const;
 
   webrtc::H264BitstreamParser h264_bitstream_parser_;
   // Reports statistics with histograms.
   void ReportInit();
   void ReportError();
 
-  ISVCEncoder* openh264_encoder_;
+  std::vector<ISVCEncoder*> encoders_;
-  // Settings that are used by this encoder.
+  std::vector<SSourcePicture> pictures_;
-  int width_;
+  std::vector<rtc::scoped_refptr<I420Buffer>> downscaled_buffers_;
-  int height_;
+  std::vector<LayerConfig> configurations_;
-  float max_frame_rate_;
+  std::vector<EncodedImage> encoded_images_;
-  uint32_t target_bps_;
+  std::vector<std::unique_ptr<uint8_t[]>> encoded_image_buffers_;
-  uint32_t max_bps_;
-  VideoCodecMode mode_;
-  // H.264 specifc parameters
-  bool frame_dropping_on_;
-  int key_frame_interval_;
-  H264PacketizationMode packetization_mode_;
 
+  VideoCodec codec_;
+  H264PacketizationMode packetization_mode_;
   size_t max_payload_size_;
   int32_t number_of_cores_;
-
-  EncodedImage encoded_image_;
-  std::unique_ptr<uint8_t[]> encoded_image_buffer_;
   EncodedImageCallback* encoded_image_callback_;
 
   bool has_reported_init_;

modules/video_coding/codecs/h264/h264_simulcast_unittest.cc

@@ -0,0 +1,99 @@
+/*
+ *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "api/test/create_simulcast_test_fixture.h"
+#include "api/test/simulcast_test_fixture.h"
+#include "modules/video_coding/codecs/h264/include/h264.h"
+#include "rtc_base/ptr_util.h"
+#include "test/function_video_decoder_factory.h"
+#include "test/function_video_encoder_factory.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace test {
+
+namespace {
+std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() {
+  std::unique_ptr<VideoEncoderFactory> encoder_factory =
+      rtc::MakeUnique<FunctionVideoEncoderFactory>(
+          []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
+  std::unique_ptr<VideoDecoderFactory> decoder_factory =
+      rtc::MakeUnique<FunctionVideoDecoderFactory>(
+          []() { return H264Decoder::Create(); });
+  return CreateSimulcastTestFixture(std::move(encoder_factory),
+                                    std::move(decoder_factory),
+                                    SdpVideoFormat("H264"));
+}
+}  // namespace
+
+TEST(TestH264Simulcast, TestKeyFrameRequestsOnAllStreams) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestKeyFrameRequestsOnAllStreams();
+}
+
+TEST(TestH264Simulcast, TestPaddingAllStreams) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestPaddingAllStreams();
+}
+
+TEST(TestH264Simulcast, TestPaddingTwoStreams) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestPaddingTwoStreams();
+}
+
+TEST(TestH264Simulcast, TestPaddingTwoStreamsOneMaxedOut) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestPaddingTwoStreamsOneMaxedOut();
+}
+
+TEST(TestH264Simulcast, TestPaddingOneStream) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestPaddingOneStream();
+}
+
+TEST(TestH264Simulcast, TestPaddingOneStreamTwoMaxedOut) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestPaddingOneStreamTwoMaxedOut();
+}
+
+TEST(TestH264Simulcast, TestSendAllStreams) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestSendAllStreams();
+}
+
+TEST(TestH264Simulcast, TestDisablingStreams) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestDisablingStreams();
+}
+
+TEST(TestH264Simulcast, TestActiveStreams) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestActiveStreams();
+}
+
+TEST(TestH264Simulcast, TestSwitchingToOneStream) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestSwitchingToOneStream();
+}
+
+TEST(TestH264Simulcast, TestSwitchingToOneOddStream) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestSwitchingToOneOddStream();
+}
+
+TEST(TestH264Simulcast, TestStrideEncodeDecode) {
+  auto fixture = CreateSpecificSimulcastTestFixture();
+  fixture->TestStrideEncodeDecode();
+}
+
+}  // namespace test
+}  // namespace webrtc

modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc

@@ -52,7 +52,7 @@ void ConfigureSimulcast(VideoCodec* codec_settings) {
   const std::vector<webrtc::VideoStream> streams = cricket::GetSimulcastConfig(
       codec_settings->numberOfSimulcastStreams, codec_settings->width,
       codec_settings->height, kMaxBitrateBps, kBitratePriority, kMaxQp,
-      kMaxFramerateFps, /* is_screenshare = */ false);
+      kMaxFramerateFps, /* is_screenshare = */ false, true);
 
   for (size_t i = 0; i < streams.size(); ++i) {
     SimulcastStream* ss = &codec_settings->simulcastStream[i];

modules/video_coding/codecs/test/videoprocessor.cc

@@ -19,10 +19,10 @@
 #include "common_video/h264/h264_common.h"
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
 #include "modules/video_coding/include/video_codec_initializer.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "modules/video_coding/utility/default_video_bitrate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/timeutils.h"
 #include "test/gtest.h"

modules/video_coding/codecs/vp8/default_temporal_layers.cc

@@ -18,7 +18,6 @@
 #include <vector>
 
 #include "modules/include/module_common_types.h"
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"

modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc

@@ -10,8 +10,8 @@
 
 #include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
 #include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "test/field_trial.h"
 #include "test/gtest.h"
 

modules/video_coding/codecs/vp8/include/vp8_common_types.h

@@ -1,29 +0,0 @@
-/*
- *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_COMMON_TYPES_H_
-#define MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_COMMON_TYPES_H_
-
-#include "common_types.h"  // NOLINT(build/include)
-
-namespace webrtc {
-
-// Ratio allocation between temporal streams:
-// Values as required for the VP8 codec (accumulating).
-static const float
-    kVp8LayerRateAlloction[kMaxSimulcastStreams][kMaxTemporalStreams] = {
-        {1.0f, 1.0f, 1.0f, 1.0f},  // 1 layer
-        {0.6f, 1.0f, 1.0f, 1.0f},  // 2 layers {60%, 40%}
-        {0.4f, 0.6f, 1.0f, 1.0f},  // 3 layers {40%, 20%, 40%}
-        {0.25f, 0.4f, 0.6f, 1.0f}  // 4 layers {25%, 15%, 20%, 40%}
-};
-
-}  // namespace webrtc
-#endif  // MODULES_VIDEO_CODING_CODECS_VP8_INCLUDE_VP8_COMMON_TYPES_H_

modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -14,7 +14,8 @@
 
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_utility.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/ptr_util.h"
 #include "rtc_base/timeutils.h"
@@ -47,7 +48,7 @@ enum denoiserState {
 };
 
 // Greatest common divisior
-int GCD(int a, int b) {
+static int GCD(int a, int b) {
   int c = a % b;
   while (c != 0) {
     a = b;
@@ -57,53 +58,6 @@ int GCD(int a, int b) {
   return b;
 }
 
-uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec) {
-  uint32_t bitrate_sum = 0;
-  for (int i = 0; i < streams; ++i) {
-    bitrate_sum += codec.simulcastStream[i].maxBitrate;
-  }
-  return bitrate_sum;
-}
-
-int NumberOfStreams(const VideoCodec& codec) {
-  int streams =
-      codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
-  uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
-  if (simulcast_max_bitrate == 0) {
-    streams = 1;
-  }
-  return streams;
-}
-
-bool ValidSimulcastResolutions(const VideoCodec& codec, int num_streams) {
-  if (codec.width != codec.simulcastStream[num_streams - 1].width ||
-      codec.height != codec.simulcastStream[num_streams - 1].height) {
-    return false;
-  }
-  for (int i = 0; i < num_streams; ++i) {
-    if (codec.width * codec.simulcastStream[i].height !=
-        codec.height * codec.simulcastStream[i].width) {
-      return false;
-    }
-  }
-  for (int i = 1; i < num_streams; ++i) {
-    if (codec.simulcastStream[i].width !=
-        codec.simulcastStream[i - 1].width * 2) {
-      return false;
-    }
-  }
-  return true;
-}
-
-bool ValidSimulcastTemporalLayers(const VideoCodec& codec, int num_streams) {
-  for (int i = 0; i < num_streams - 1; ++i) {
-    if (codec.simulcastStream[i].numberOfTemporalLayers !=
-        codec.simulcastStream[i + 1].numberOfTemporalLayers)
-      return false;
-  }
-  return true;
-}
-
 bool GetGfBoostPercentageFromFieldTrialGroup(int* boost_percentage) {
   std::string group = webrtc::field_trial::FindFullName(kVp8GfBoostFieldTrial);
   if (group.empty())
@@ -369,12 +323,13 @@ int LibvpxVp8Encoder::InitEncode(const VideoCodec* inst,
     return retVal;
   }
 
-  int number_of_streams = NumberOfStreams(*inst);
+  int number_of_streams = SimulcastUtility::NumberOfSimulcastStreams(*inst);
   bool doing_simulcast = (number_of_streams > 1);
 
-  if (doing_simulcast &&
+  if (doing_simulcast && (!SimulcastUtility::ValidSimulcastResolutions(
-      (!ValidSimulcastResolutions(*inst, number_of_streams) ||
+                              *inst, number_of_streams) ||
-       !ValidSimulcastTemporalLayers(*inst, number_of_streams))) {
+                          !SimulcastUtility::ValidSimulcastTemporalLayers(
+                              *inst, number_of_streams))) {
     return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
   }
 

modules/video_coding/codecs/vp8/libvpx_vp8_simulcast_test.cc

@@ -30,7 +30,8 @@ std::unique_ptr<SimulcastTestFixture> CreateSpecificSimulcastTestFixture() {
       rtc::MakeUnique<FunctionVideoDecoderFactory>(
           []() { return VP8Decoder::Create(); });
   return CreateSimulcastTestFixture(std::move(encoder_factory),
-                                    std::move(decoder_factory));
+                                    std::move(decoder_factory),
+                                    SdpVideoFormat("VP8"));
 }
 }  // namespace
 

modules/video_coding/codecs/vp8/screenshare_layers.cc

@@ -37,7 +37,8 @@ constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
 // been exceeded. This prevents needless keyframe requests.
 const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
 
-ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, Clock* clock)
+ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
+                                     Clock* clock)
     : clock_(clock),
       number_of_temporal_layers_(
           std::min(kMaxNumTemporalLayers, num_temporal_layers)),

modules/video_coding/codecs/vp8/screenshare_layers.h

@@ -28,7 +28,8 @@ class ScreenshareLayers : public TemporalLayers {
   static const double kAcceptableTargetOvershoot;
   static const int kMaxFrameIntervalMs;
 
-  ScreenshareLayers(int num_temporal_layers, Clock* clock);
+  ScreenshareLayers(int num_temporal_layers,
+                    Clock* clock);
   virtual ~ScreenshareLayers();
 
   // Returns the recommended VP8 encode flags needed. May refresh the decoder

modules/video_coding/codecs/vp8/temporal_layers.cc

@@ -16,7 +16,6 @@
 
 #include "modules/include/module_common_types.h"
 #include "modules/video_coding/codecs/vp8/default_temporal_layers.h"
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
 #include "modules/video_coding/codecs/vp8/screenshare_layers.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "rtc_base/checks.h"

modules/video_coding/include/video_codec_interface.h

@@ -70,6 +70,7 @@ struct CodecSpecificInfoGeneric {
 
 struct CodecSpecificInfoH264 {
   H264PacketizationMode packetization_mode;
+  uint8_t simulcast_idx;
 };
 
 union CodecSpecificInfoUnion {
@@ -83,7 +84,9 @@ union CodecSpecificInfoUnion {
 // must be fitted with a copy-constructor. This is because it is copied
 // in the copy-constructor of VCMEncodedFrame.
 struct CodecSpecificInfo {
-  CodecSpecificInfo() : codecType(kVideoCodecUnknown), codec_name(nullptr) {}
+  CodecSpecificInfo() : codecType(kVideoCodecUnknown), codec_name(nullptr) {
+    memset(&codecSpecific, 0, sizeof(codecSpecific));
+  }
   VideoCodecType codecType;
   const char* codec_name;
   CodecSpecificInfoUnion codecSpecific;

modules/video_coding/media_opt_util.cc

@@ -18,10 +18,10 @@
 #include <limits>
 
 #include "modules/include/module_common_types.h"
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
 #include "modules/video_coding/fec_rate_table.h"
 #include "modules/video_coding/include/video_coding_defines.h"
 #include "modules/video_coding/nack_fec_tables.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 
 namespace webrtc {
 // Max value of loss rates in off-line model
@@ -400,8 +400,7 @@ bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) {
 int VCMFecMethod::BitsPerFrame(const VCMProtectionParameters* parameters) {
   // When temporal layers are available FEC will only be applied on the base
   // layer.
-  const float bitRateRatio =
+  const float bitRateRatio = kLayerRateAllocation[parameters->numLayers - 1][0];
-      kVp8LayerRateAlloction[parameters->numLayers - 1][0];
   float frameRateRatio = powf(1 / 2.0, parameters->numLayers - 1);
   float bitRate = parameters->bitRate * bitRateRatio;
   float frameRate = parameters->frameRate * frameRateRatio;

modules/video_coding/utility/simulcast_rate_allocator.cc

@@ -8,14 +8,14 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 
 #include <algorithm>
 #include <memory>
 #include <utility>
 #include <vector>
 
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
+#include "common_types.h"  // NOLINT(build/include)
 #include "rtc_base/checks.h"
 
 namespace webrtc {
@@ -190,7 +190,7 @@ std::vector<uint32_t> SimulcastRateAllocator::DefaultTemporalLayerAllocation(
   std::vector<uint32_t> bitrates;
   for (size_t i = 0; i < num_temporal_layers; ++i) {
     float layer_bitrate =
-        bitrate_kbps * kVp8LayerRateAlloction[num_temporal_layers - 1][i];
+        bitrate_kbps * kLayerRateAllocation[num_temporal_layers - 1][i];
     bitrates.push_back(static_cast<uint32_t>(layer_bitrate + 0.5));
   }
 
@@ -235,7 +235,8 @@ const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const {
 
 int SimulcastRateAllocator::NumTemporalStreams(size_t simulcast_id) const {
   return std::max<uint8_t>(
-      1, codec_.numberOfSimulcastStreams == 0
+      1,
+      codec_.codecType == kVideoCodecVP8 && codec_.numberOfSimulcastStreams == 0
           ? codec_.VP8().numberOfTemporalLayers
           : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers);
 }

modules/video_coding/utility/simulcast_rate_allocator.h

@@ -8,8 +8,8 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_RATE_ALLOCATOR_H_
+#ifndef MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_
-#define MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_RATE_ALLOCATOR_H_
+#define MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_
 
 #include <stdint.h>
 
@@ -20,11 +20,20 @@
 #include "api/video_codecs/video_encoder.h"
 #include "common_types.h"  // NOLINT(build/include)
 #include "common_video/include/video_bitrate_allocator.h"
-#include "modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "rtc_base/constructormagic.h"
 
 namespace webrtc {
 
+// Ratio allocation between temporal streams:
+// Values as required for the VP8 codec (accumulating).
+static const float
+    kLayerRateAllocation[kMaxSimulcastStreams][kMaxTemporalStreams] = {
+        {1.0f, 1.0f, 1.0f, 1.0f},  // 1 layer
+        {0.6f, 1.0f, 1.0f, 1.0f},  // 2 layers {60%, 40%}
+        {0.4f, 0.6f, 1.0f, 1.0f},  // 3 layers {40%, 20%, 40%}
+        {0.25f, 0.4f, 0.6f, 1.0f}  // 4 layers {25%, 15%, 20%, 40%}
+};
+
 class SimulcastRateAllocator : public VideoBitrateAllocator {
  public:
   explicit SimulcastRateAllocator(const VideoCodec& codec);
@@ -58,4 +67,4 @@ class SimulcastRateAllocator : public VideoBitrateAllocator {
 
 }  // namespace webrtc
 
-#endif  // MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_RATE_ALLOCATOR_H_
+#endif  // MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_

modules/video_coding/utility/simulcast_rate_allocator_unittest.cc

@@ -8,13 +8,15 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 
 #include <limits>
 #include <memory>
 #include <utility>
 #include <vector>
 
+#include "modules/video_coding/codecs/vp8/temporal_layers.h"
+
 #include "test/gmock.h"
 #include "test/gtest.h"
 

modules/video_coding/utility/simulcast_test_fixture_impl.cc

@@ -8,7 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "modules/video_coding/codecs/vp8/simulcast_test_fixture_impl.h"
+#include "modules/video_coding/utility/simulcast_test_fixture_impl.h"
 
 #include <algorithm>
 #include <map>
@@ -18,8 +18,6 @@
 #include "api/video_codecs/sdp_video_format.h"
 #include "common_video/include/video_frame.h"
 #include "common_video/libyuv/include/webrtc_libyuv.h"
-#include "modules/video_coding/codecs/vp8/include/vp8.h"
-#include "modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "modules/video_coding/include/video_coding_defines.h"
 #include "rtc_base/checks.h"
 #include "test/gtest.h"
@@ -44,6 +42,7 @@ const int kMaxBitrates[kNumberOfSimulcastStreams] = {150, 600, 1200};
 const int kMinBitrates[kNumberOfSimulcastStreams] = {50, 150, 600};
 const int kTargetBitrates[kNumberOfSimulcastStreams] = {100, 450, 1000};
 const int kDefaultTemporalLayerProfile[3] = {3, 3, 3};
+const int kNoTemporalLayerProfile[3] = {0, 0, 0};
 
 template <typename T>
 void SetExpectedValues3(T value0, T value1, T value2, T* expected_values) {
@@ -61,15 +60,15 @@ enum PlaneType {
 
 }  // namespace
 
-class SimulcastTestFixtureImpl::Vp8TestEncodedImageCallback
+class SimulcastTestFixtureImpl::TestEncodedImageCallback
     : public EncodedImageCallback {
  public:
-  Vp8TestEncodedImageCallback() : picture_id_(-1) {
+  TestEncodedImageCallback() {
     memset(temporal_layer_, -1, sizeof(temporal_layer_));
     memset(layer_sync_, false, sizeof(layer_sync_));
   }
 
-  ~Vp8TestEncodedImageCallback() {
+  ~TestEncodedImageCallback() {
     delete[] encoded_key_frame_._buffer;
     delete[] encoded_frame_._buffer;
   }
@@ -77,8 +76,15 @@ class SimulcastTestFixtureImpl::Vp8TestEncodedImageCallback
   virtual Result OnEncodedImage(const EncodedImage& encoded_image,
                                 const CodecSpecificInfo* codec_specific_info,
                                 const RTPFragmentationHeader* fragmentation) {
+    uint16_t simulcast_idx = 0;
+    bool is_vp8 = (codec_specific_info->codecType == kVideoCodecVP8);
+    if (is_vp8) {
+      simulcast_idx = codec_specific_info->codecSpecific.VP8.simulcastIdx;
+    } else {
+      simulcast_idx = codec_specific_info->codecSpecific.H264.simulcast_idx;
+    }
     // Only store the base layer.
-    if (codec_specific_info->codecSpecific.VP8.simulcastIdx == 0) {
+    if (simulcast_idx) {
       if (encoded_image._frameType == kVideoFrameKey) {
         delete[] encoded_key_frame_._buffer;
         encoded_key_frame_._buffer = new uint8_t[encoded_image._size];
@@ -97,17 +103,18 @@ class SimulcastTestFixtureImpl::Vp8TestEncodedImageCallback
                encoded_image._length);
       }
     }
+    if (is_vp8) {
       layer_sync_[codec_specific_info->codecSpecific.VP8.simulcastIdx] =
           codec_specific_info->codecSpecific.VP8.layerSync;
       temporal_layer_[codec_specific_info->codecSpecific.VP8.simulcastIdx] =
           codec_specific_info->codecSpecific.VP8.temporalIdx;
+    }
     return Result(Result::OK, encoded_image._timeStamp);
   }
-  void GetLastEncodedFrameInfo(int* picture_id,
+  // This method only makes sense for VP8.
-                               int* temporal_layer,
+  void GetLastEncodedFrameInfo(int* temporal_layer,
                                bool* layer_sync,
                                int stream) {
-    *picture_id = picture_id_;
     *temporal_layer = temporal_layer_[stream];
     *layer_sync = layer_sync_[stream];
   }
@@ -121,15 +128,14 @@ class SimulcastTestFixtureImpl::Vp8TestEncodedImageCallback
  private:
   EncodedImage encoded_key_frame_;
   EncodedImage encoded_frame_;
-  int picture_id_;
   int temporal_layer_[kNumberOfSimulcastStreams];
   bool layer_sync_[kNumberOfSimulcastStreams];
 };
 
-class SimulcastTestFixtureImpl::Vp8TestDecodedImageCallback
+class SimulcastTestFixtureImpl::TestDecodedImageCallback
     : public DecodedImageCallback {
  public:
-  Vp8TestDecodedImageCallback() : decoded_frames_(0) {}
+  TestDecodedImageCallback() : decoded_frames_(0) {}
   int32_t Decoded(VideoFrame& decoded_image) override {
     rtc::scoped_refptr<I420BufferInterface> i420_buffer =
         decoded_image.video_frame_buffer()->ToI420();
@@ -198,7 +204,9 @@ void ConfigureStream(int width,
   stream->maxBitrate = max_bitrate;
   stream->minBitrate = min_bitrate;
   stream->targetBitrate = target_bitrate;
+  if (num_temporal_layers >= 0) {
     stream->numberOfTemporalLayers = num_temporal_layers;
+  }
   stream->qpMax = 45;
   stream->active = true;
 }
@@ -207,10 +215,11 @@ void ConfigureStream(int width,
 
 void SimulcastTestFixtureImpl::DefaultSettings(
     VideoCodec* settings,
-    const int* temporal_layer_profile) {
+    const int* temporal_layer_profile,
+    VideoCodecType codec_type) {
   RTC_CHECK(settings);
   memset(settings, 0, sizeof(VideoCodec));
-  settings->codecType = kVideoCodecVP8;
+  settings->codecType = codec_type;
   // 96 to 127 dynamic payload types for video codecs
   settings->plType = 120;
   settings->startBitrate = 300;
@@ -233,18 +242,26 @@ void SimulcastTestFixtureImpl::DefaultSettings(
   ConfigureStream(kDefaultWidth, kDefaultHeight, kMaxBitrates[2],
                   kMinBitrates[2], kTargetBitrates[2],
                   &settings->simulcastStream[2], temporal_layer_profile[2]);
+    if (codec_type == kVideoCodecVP8) {
       settings->VP8()->denoisingOn = true;
       settings->VP8()->automaticResizeOn = false;
       settings->VP8()->frameDroppingOn = true;
       settings->VP8()->keyFrameInterval = 3000;
+    } else {
+      settings->H264()->frameDroppingOn = true;
+      settings->H264()->keyFrameInterval = 3000;
+    }
 }
 
 SimulcastTestFixtureImpl::SimulcastTestFixtureImpl(
     std::unique_ptr<VideoEncoderFactory> encoder_factory,
-    std::unique_ptr<VideoDecoderFactory> decoder_factory) {
+    std::unique_ptr<VideoDecoderFactory> decoder_factory,
-  encoder_ = encoder_factory->CreateVideoEncoder(SdpVideoFormat("VP8"));
+    SdpVideoFormat video_format)
-  decoder_ = decoder_factory->CreateVideoDecoder(SdpVideoFormat("VP8"));
+    : codec_type_(PayloadStringToCodecType(video_format.name)) {
-  SetUpCodec(kDefaultTemporalLayerProfile);
+  encoder_ = encoder_factory->CreateVideoEncoder(video_format);
+  decoder_ = decoder_factory->CreateVideoDecoder(video_format);
+  SetUpCodec(codec_type_ == kVideoCodecVP8 ? kDefaultTemporalLayerProfile
+    : kNoTemporalLayerProfile);
 }
 
 SimulcastTestFixtureImpl::~SimulcastTestFixtureImpl() {
@@ -255,7 +272,7 @@ SimulcastTestFixtureImpl::~SimulcastTestFixtureImpl() {
 void SimulcastTestFixtureImpl::SetUpCodec(const int* temporal_layer_profile) {
   encoder_->RegisterEncodeCompleteCallback(&encoder_callback_);
   decoder_->RegisterDecodeCompleteCallback(&decoder_callback_);
-  DefaultSettings(&settings_, temporal_layer_profile);
+  DefaultSettings(&settings_, temporal_layer_profile, codec_type_);
   SetUpRateAllocator();
   EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200));
   EXPECT_EQ(0, decoder_->InitDecode(&settings_, 1));
@@ -361,16 +378,14 @@ void SimulcastTestFixtureImpl::ExpectStreams(FrameType frame_type,
 }
 
 void SimulcastTestFixtureImpl::VerifyTemporalIdxAndSyncForAllSpatialLayers(
-    Vp8TestEncodedImageCallback* encoder_callback,
+    TestEncodedImageCallback* encoder_callback,
     const int* expected_temporal_idx,
     const bool* expected_layer_sync,
     int num_spatial_layers) {
-  int picture_id = -1;
   int temporal_layer = -1;
   bool layer_sync = false;
   for (int i = 0; i < num_spatial_layers; i++) {
-    encoder_callback->GetLastEncodedFrameInfo(&picture_id, &temporal_layer,
+    encoder_callback->GetLastEncodedFrameInfo(&temporal_layer, &layer_sync, i);
-                                              &layer_sync, i);
     EXPECT_EQ(expected_temporal_idx[i], temporal_layer);
     EXPECT_EQ(expected_layer_sync[i], layer_sync);
   }
@@ -558,9 +573,15 @@ void SimulcastTestFixtureImpl::TestActiveStreams() {
 }
 
 void SimulcastTestFixtureImpl::SwitchingToOneStream(int width, int height) {
+  const int* temporal_layer_profile = nullptr;
   // Disable all streams except the last and set the bitrate of the last to
   // 100 kbps. This verifies the way GTP switches to screenshare mode.
+  if (codec_type_ == kVideoCodecVP8) {
     settings_.VP8()->numberOfTemporalLayers = 1;
+    temporal_layer_profile = kDefaultTemporalLayerProfile;
+  } else {
+    temporal_layer_profile = kNoTemporalLayerProfile;
+  }
   settings_.maxBitrate = 100;
   settings_.startBitrate = 100;
   settings_.width = width;
@@ -605,7 +626,7 @@ void SimulcastTestFixtureImpl::SwitchingToOneStream(int width, int height) {
   EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types));
 
   // Switch back.
-  DefaultSettings(&settings_, kDefaultTemporalLayerProfile);
+  DefaultSettings(&settings_, temporal_layer_profile, codec_type_);
   // Start at the lowest bitrate for enabling base stream.
   settings_.startBitrate = kMinBitrates[0];
   SetUpRateAllocator();
@@ -636,7 +657,8 @@ void SimulcastTestFixtureImpl::TestSwitchingToOneSmallStream() {
 // 3-3-3 pattern: 3 temporal layers for all spatial streams, so same
 // temporal_layer id and layer_sync is expected for all streams.
 void SimulcastTestFixtureImpl::TestSpatioTemporalLayers333PatternEncoder() {
-  Vp8TestEncodedImageCallback encoder_callback;
+  EXPECT_EQ(codec_type_, kVideoCodecVP8);
+  TestEncodedImageCallback encoder_callback;
   encoder_->RegisterEncodeCompleteCallback(&encoder_callback);
   SetRates(kMaxBitrates[2], 30);  // To get all three streams.
 
@@ -703,9 +725,10 @@ void SimulcastTestFixtureImpl::TestSpatioTemporalLayers333PatternEncoder() {
 // Since CodecSpecificInfoVP8.temporalIdx is uint8_t, this will wrap to 255.
 // TODO(marpan): Although this seems safe for now, we should fix this.
 void SimulcastTestFixtureImpl::TestSpatioTemporalLayers321PatternEncoder() {
+  EXPECT_EQ(codec_type_, kVideoCodecVP8);
   int temporal_layer_profile[3] = {3, 2, 1};
   SetUpCodec(temporal_layer_profile);
-  Vp8TestEncodedImageCallback encoder_callback;
+  TestEncodedImageCallback encoder_callback;
   encoder_->RegisterEncodeCompleteCallback(&encoder_callback);
   SetRates(kMaxBitrates[2], 30);  // To get all three streams.
 
@@ -761,8 +784,8 @@ void SimulcastTestFixtureImpl::TestSpatioTemporalLayers321PatternEncoder() {
 }
 
 void SimulcastTestFixtureImpl::TestStrideEncodeDecode() {
-  Vp8TestEncodedImageCallback encoder_callback;
+  TestEncodedImageCallback encoder_callback;
-  Vp8TestDecodedImageCallback decoder_callback;
+  TestDecodedImageCallback decoder_callback;
   encoder_->RegisterEncodeCompleteCallback(&encoder_callback);
   decoder_->RegisterDecodeCompleteCallback(&decoder_callback);
 

modules/video_coding/utility/simulcast_test_fixture_impl.h

@@ -8,8 +8,8 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#ifndef MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_TEST_FIXTURE_IMPL_H_
+#ifndef MODULES_VIDEO_CODING_UTILITY_SIMULCAST_TEST_FIXTURE_IMPL_H_
-#define MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_TEST_FIXTURE_IMPL_H_
+#define MODULES_VIDEO_CODING_UTILITY_SIMULCAST_TEST_FIXTURE_IMPL_H_
 
 #include <memory>
 #include <vector>
@@ -20,7 +20,7 @@
 #include "api/video_codecs/video_decoder_factory.h"
 #include "api/video_codecs/video_encoder_factory.h"
 #include "common_types.h"  // NOLINT(build/include)
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "modules/video_coding/include/mock/mock_video_codec_interface.h"
 
 namespace webrtc {
@@ -30,7 +30,8 @@ class SimulcastTestFixtureImpl final : public SimulcastTestFixture {
  public:
   SimulcastTestFixtureImpl(
       std::unique_ptr<VideoEncoderFactory> encoder_factory,
-      std::unique_ptr<VideoDecoderFactory> decoder_factory);
+      std::unique_ptr<VideoDecoderFactory> decoder_factory,
+      SdpVideoFormat video_format);
   ~SimulcastTestFixtureImpl() final;
 
   // Implements SimulcastTestFixture.
@@ -51,11 +52,12 @@ class SimulcastTestFixtureImpl final : public SimulcastTestFixture {
   void TestStrideEncodeDecode() override;
 
   static void DefaultSettings(VideoCodec* settings,
-                              const int* temporal_layer_profile);
+                              const int* temporal_layer_profile,
+                              VideoCodecType codec_type);
 
  private:
-  class Vp8TestEncodedImageCallback;
+  class TestEncodedImageCallback;
-  class Vp8TestDecodedImageCallback;
+  class TestDecodedImageCallback;
 
   void SetUpCodec(const int* temporal_layer_profile);
   void SetUpRateAllocator();
@@ -66,7 +68,7 @@ class SimulcastTestFixtureImpl final : public SimulcastTestFixture {
                      const std::vector<bool> expected_streams_active);
   void ExpectStreams(FrameType frame_type, int expected_video_streams);
   void VerifyTemporalIdxAndSyncForAllSpatialLayers(
-      Vp8TestEncodedImageCallback* encoder_callback,
+      TestEncodedImageCallback* encoder_callback,
       const int* expected_temporal_idx,
       const bool* expected_layer_sync,
       int num_spatial_layers);
@@ -80,9 +82,10 @@ class SimulcastTestFixtureImpl final : public SimulcastTestFixture {
   rtc::scoped_refptr<I420Buffer> input_buffer_;
   std::unique_ptr<VideoFrame> input_frame_;
   std::unique_ptr<SimulcastRateAllocator> rate_allocator_;
+  VideoCodecType codec_type_;
 };
 
 }  // namespace test
 }  // namespace webrtc
 
-#endif  // MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_TEST_FIXTURE_IMPL_H_
+#endif  // MODULES_VIDEO_CODING_UTILITY_SIMULCAST_TEST_FIXTURE_IMPL_H_

modules/video_coding/utility/simulcast_utility.cc

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/simulcast_utility.h"
+
+namespace webrtc {
+
+uint32_t SimulcastUtility::SumStreamMaxBitrate(int streams,
+                                               const VideoCodec& codec) {
+  uint32_t bitrate_sum = 0;
+  for (int i = 0; i < streams; ++i) {
+    bitrate_sum += codec.simulcastStream[i].maxBitrate;
+  }
+  return bitrate_sum;
+}
+
+int SimulcastUtility::NumberOfSimulcastStreams(const VideoCodec& codec) {
+  int streams =
+      codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
+  uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
+  if (simulcast_max_bitrate == 0) {
+    streams = 1;
+  }
+  return streams;
+}
+
+bool SimulcastUtility::ValidSimulcastResolutions(const VideoCodec& codec,
+                                                 int num_streams) {
+  if (codec.width != codec.simulcastStream[num_streams - 1].width ||
+      codec.height != codec.simulcastStream[num_streams - 1].height) {
+    return false;
+  }
+  for (int i = 0; i < num_streams; ++i) {
+    if (codec.width * codec.simulcastStream[i].height !=
+        codec.height * codec.simulcastStream[i].width) {
+      return false;
+    }
+  }
+  for (int i = 1; i < num_streams; ++i) {
+    if (codec.simulcastStream[i].width !=
+        codec.simulcastStream[i - 1].width * 2) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool SimulcastUtility::ValidSimulcastTemporalLayers(const VideoCodec& codec,
+                                                    int num_streams) {
+  for (int i = 0; i < num_streams - 1; ++i) {
+    if (codec.simulcastStream[i].numberOfTemporalLayers !=
+        codec.simulcastStream[i + 1].numberOfTemporalLayers)
+      return false;
+  }
+  return true;
+}
+
+}  // namespace webrtc

modules/video_coding/utility/simulcast_utility.h

@@ -0,0 +1,30 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_UTILITY_SIMULCAST_UTILITY_H_
+#define MODULES_VIDEO_CODING_UTILITY_SIMULCAST_UTILITY_H_
+
+#include "api/video_codecs/video_encoder.h"
+
+namespace webrtc {
+
+class SimulcastUtility {
+ public:
+  static uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec);
+  static int NumberOfSimulcastStreams(const VideoCodec& codec);
+  static bool ValidSimulcastResolutions(const VideoCodec& codec,
+                                        int num_streams);
+  static bool ValidSimulcastTemporalLayers(const VideoCodec& codec,
+                                           int num_streams);
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_UTILITY_SIMULCAST_UTILITY_H_

modules/video_coding/video_codec_initializer.cc

@@ -13,14 +13,13 @@
 #include "api/video_codecs/video_encoder.h"
 #include "common_types.h"  // NOLINT(build/include)
 #include "common_video/include/video_bitrate_allocator.h"
-#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
-#include "modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "modules/video_coding/codecs/vp9/svc_config.h"
 #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
 #include "modules/video_coding/include/video_coding_defines.h"
 #include "modules/video_coding/utility/default_video_bitrate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "rtc_base/logging.h"
+#include "rtc_base/system/fallthrough.h"
 #include "system_wrappers/include/clock.h"
 
 namespace webrtc {
@@ -53,7 +52,8 @@ VideoCodecInitializer::CreateBitrateAllocator(const VideoCodec& codec) {
 
   switch (codec.codecType) {
     case kVideoCodecVP8:
-      // Set up default VP8 temporal layer factory, if not provided.
+      RTC_FALLTHROUGH();
+    case kVideoCodecH264:
       rate_allocator.reset(new SimulcastRateAllocator(codec));
       break;
     case kVideoCodecVP9:

modules/video_coding/video_coding_impl.cc

@@ -16,7 +16,6 @@
 #include "common_types.h"  // NOLINT(build/include)
 #include "common_video/include/video_bitrate_allocator.h"
 #include "common_video/libyuv/include/webrtc_libyuv.h"
-#include "modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "modules/video_coding/encoded_frame.h"
 #include "modules/video_coding/include/video_codec_initializer.h"
 #include "modules/video_coding/include/video_codec_interface.h"
@@ -105,17 +104,16 @@ class VideoCodingModuleImpl : public VideoCodingModule {
   int32_t RegisterSendCodec(const VideoCodec* sendCodec,
                             uint32_t numberOfCores,
                             uint32_t maxPayloadSize) override {
-    if (sendCodec != nullptr && sendCodec->codecType == kVideoCodecVP8) {
+    if (sendCodec != nullptr && ((sendCodec->codecType == kVideoCodecVP8) ||
-      // Set up a rate allocator and temporal layers factory for this vp8
+                                 (sendCodec->codecType == kVideoCodecH264))) {
+      // Set up a rate allocator and temporal layers factory for this codec
       // instance. The codec impl will have a raw pointer to the TL factory,
       // and will call it when initializing. Since this can happen
       // asynchronously keep the instance alive until destruction or until a
       // new send codec is registered.
-      VideoCodec vp8_codec = *sendCodec;
+      VideoCodec codec = *sendCodec;
-      rate_allocator_ =
+      rate_allocator_ = VideoCodecInitializer::CreateBitrateAllocator(codec);
-          VideoCodecInitializer::CreateBitrateAllocator(vp8_codec);
+      return sender_.RegisterSendCodec(&codec, numberOfCores, maxPayloadSize);
-      return sender_.RegisterSendCodec(&vp8_codec, numberOfCores,
-                                       maxPayloadSize);
     }
     return sender_.RegisterSendCodec(sendCodec, numberOfCores, maxPayloadSize);
   }

modules/video_coding/video_sender.cc

@@ -8,6 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+
 #include <algorithm>  // std::max
 
 #include "common_types.h"  // NOLINT(build/include)
@@ -136,7 +137,8 @@ void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
     }
     return;
   }
-  _codecDataBase.RegisterExternalEncoder(externalEncoder, internalSource);
+  _codecDataBase.RegisterExternalEncoder(externalEncoder,
+                                         internalSource);
 }
 
 EncoderParameters VideoSender::UpdateEncoderParameters(
@@ -289,7 +291,8 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
       RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
       return VCM_PARAMETER_ERROR;
     }
-    converted_frame = VideoFrame(converted_buffer, converted_frame.timestamp(),
+    converted_frame = VideoFrame(converted_buffer,
+                                 converted_frame.timestamp(),
                                  converted_frame.render_time_ms(),
                                  converted_frame.rotation());
   }

modules/video_coding/video_sender_unittest.cc

@@ -13,13 +13,12 @@
 
 #include "api/video/i420_buffer.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
-#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
 #include "modules/video_coding/codecs/vp8/temporal_layers.h"
 #include "modules/video_coding/include/mock/mock_vcm_callbacks.h"
 #include "modules/video_coding/include/mock/mock_video_codec_interface.h"
 #include "modules/video_coding/include/video_coding.h"
 #include "modules/video_coding/utility/default_video_bitrate_allocator.h"
+#include "modules/video_coding/utility/simulcast_rate_allocator.h"
 #include "modules/video_coding/video_coding_impl.h"
 #include "system_wrappers/include/clock.h"
 #include "test/frame_generator.h"
@@ -472,9 +471,9 @@ class TestVideoSenderWithVp8 : public TestVideoSender {
 #define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy
 #endif
 TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) {
-  const int low_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][0];
+  const int low_b = codec_bitrate_kbps_ * kLayerRateAllocation[2][0];
-  const int mid_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][1];
+  const int mid_b = codec_bitrate_kbps_ * kLayerRateAllocation[2][1];
-  const int high_b = codec_bitrate_kbps_ * kVp8LayerRateAlloction[2][2];
+  const int high_b = codec_bitrate_kbps_ * kLayerRateAllocation[2][2];
   {
     Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}};
     EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected));

test/BUILD.gn

@@ -577,7 +577,7 @@ rtc_source_set("test_common") {
     "../modules/video_coding:video_coding_utility",
     "../modules/video_coding:webrtc_h264",
     "../modules/video_coding:webrtc_multiplex",
-    "../modules/video_coding:webrtc_vp8_helpers",
+    "../modules/video_coding:webrtc_vp8",
     "../modules/video_coding:webrtc_vp9",
     "../rtc_base:checks",
     "../rtc_base:rtc_base_approved",

video/BUILD.gn

@@ -166,6 +166,7 @@ if (rtc_include_tests) {
       "../media:rtc_internal_video_codecs",
       "../modules/audio_mixer:audio_mixer_impl",
       "../modules/rtp_rtcp",
+      "../modules/video_coding:video_coding",
       "../modules/video_coding:webrtc_h264",
       "../modules/video_coding:webrtc_multiplex",
       "../modules/video_coding:webrtc_vp8",
@@ -401,7 +402,7 @@ if (rtc_include_tests) {
       "../modules/video_coding:video_coding_utility",
       "../modules/video_coding:webrtc_h264",
       "../modules/video_coding:webrtc_multiplex",
-      "../modules/video_coding:webrtc_vp8_helpers",
+      "../modules/video_coding:webrtc_vp8",
       "../modules/video_coding:webrtc_vp9",
       "../rtc_base:checks",
       "../rtc_base:rate_limiter",

video/payload_router.cc

@@ -78,6 +78,7 @@ void CopyCodecSpecific(const CodecSpecificInfo* info, RTPVideoHeader* rtp) {
     case kVideoCodecH264:
       rtp->codecHeader.H264.packetization_mode =
           info->codecSpecific.H264.packetization_mode;
+      rtp->simulcastIdx = info->codecSpecific.H264.simulcast_idx;
       return;
     case kVideoCodecMultiplex:
     case kVideoCodecGeneric:

video/picture_id_tests.cc

@@ -10,6 +10,7 @@
 #include "media/engine/internalencoderfactory.h"
 #include "media/engine/simulcast_encoder_adapter.h"
 #include "modules/rtp_rtcp/source/rtp_format.h"
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "rtc_base/numerics/safe_conversions.h"
 #include "rtc_base/numerics/sequence_number_util.h"

video/send_statistics_proxy.cc

@@ -429,9 +429,22 @@ void SendStatisticsProxy::UmaSamplesContainer::UpdateHistograms(
     int qp_h264 = it.second.h264.Avg(kMinRequiredMetricsSamples);
     if (qp_h264 != -1) {
       int spatial_idx = it.first;
-      RTC_DCHECK_EQ(-1, spatial_idx);
+      if (spatial_idx == -1) {
-      RTC_HISTOGRAMS_COUNTS_100(kIndex, uma_prefix_ + "Encoded.Qp.H264",
+        RTC_HISTOGRAMS_COUNTS_200(kIndex, uma_prefix_ + "Encoded.Qp.H264",
                                   qp_h264);
+      } else if (spatial_idx == 0) {
+        RTC_HISTOGRAMS_COUNTS_200(kIndex, uma_prefix_ + "Encoded.Qp.H264.S0",
+                                  qp_h264);
+      } else if (spatial_idx == 1) {
+        RTC_HISTOGRAMS_COUNTS_200(kIndex, uma_prefix_ + "Encoded.Qp.H264.S1",
+                                  qp_h264);
+      } else if (spatial_idx == 2) {
+        RTC_HISTOGRAMS_COUNTS_200(kIndex, uma_prefix_ + "Encoded.Qp.H264.S2",
+                                  qp_h264);
+      } else {
+        RTC_LOG(LS_WARNING)
+            << "QP stats not recorded for H264 spatial idx " << spatial_idx;
+      }
     }
   }
 
@@ -858,6 +871,8 @@ void SendStatisticsProxy::OnSendEncodedImage(
   if (codec_info) {
     if (codec_info->codecType == kVideoCodecVP8) {
       simulcast_idx = codec_info->codecSpecific.VP8.simulcastIdx;
+    } else if (codec_info->codecType == kVideoCodecH264) {
+      simulcast_idx = codec_info->codecSpecific.H264.simulcast_idx;
     } else if (codec_info->codecType == kVideoCodecGeneric) {
       simulcast_idx = codec_info->codecSpecific.generic.simulcast_idx;
     }
@@ -906,7 +921,9 @@ void SendStatisticsProxy::OnSendEncodedImage(
                 : codec_info->codecSpecific.VP9.spatial_idx;
         uma_container_->qp_counters_[spatial_idx].vp9.Add(encoded_image.qp_);
       } else if (codec_info->codecType == kVideoCodecH264) {
-        int spatial_idx = -1;
+        int spatial_idx = (rtp_config_.ssrcs.size() == 1)
+                              ? -1
+                              : static_cast<int>(simulcast_idx);
         uma_container_->qp_counters_[spatial_idx].h264.Add(encoded_image.qp_);
       }
     }

video/send_statistics_proxy_unittest.cc

@@ -1253,12 +1253,18 @@ TEST_F(SendStatisticsProxyTest, VerifyQpHistogramStats_H264) {
   codec_info.codecType = kVideoCodecH264;
 
   for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
+    codec_info.codecSpecific.H264.simulcast_idx = 0;
     encoded_image.qp_ = kQpIdx0;
     statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
+    codec_info.codecSpecific.H264.simulcast_idx = 1;
+    encoded_image.qp_ = kQpIdx1;
+    statistics_proxy_->OnSendEncodedImage(encoded_image, &codec_info);
   }
   statistics_proxy_.reset();
-  EXPECT_EQ(1, metrics::NumSamples("WebRTC.Video.Encoded.Qp.H264"));
+  EXPECT_EQ(1, metrics::NumSamples("WebRTC.Video.Encoded.Qp.H264.S0"));
-  EXPECT_EQ(1, metrics::NumEvents("WebRTC.Video.Encoded.Qp.H264", kQpIdx0));
+  EXPECT_EQ(1, metrics::NumEvents("WebRTC.Video.Encoded.Qp.H264.S0", kQpIdx0));
+  EXPECT_EQ(1, metrics::NumSamples("WebRTC.Video.Encoded.Qp.H264.S1"));
+  EXPECT_EQ(1, metrics::NumEvents("WebRTC.Video.Encoded.Qp.H264.S1", kQpIdx1));
 }
 
 TEST_F(SendStatisticsProxyTest,

video/video_quality_test.cc

@@ -27,7 +27,6 @@
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
-#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "rtc_base/cpu_time.h"
 #include "rtc_base/flags.h"