diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc
index bee7b23c4e..3af022ab17 100644
--- a/media/engine/simulcast_encoder_adapter.cc
+++ b/media/engine/simulcast_encoder_adapter.cc
@@ -344,20 +344,24 @@ int SimulcastEncoderAdapter::InitEncode(
 
   // Two distinct scenarios:
   // * Singlecast (total_streams_count == 1) or simulcast with simulcast-capable
-  //   underlaying encoder implementation. SEA operates in bypass mode: original
-  //   settings are passed to the underlaying encoder, frame encode complete
-  //   callback is not intercepted.
+  //   underlaying encoder implementation if active_streams_count > 1. SEA
+  //   operates in bypass mode: original settings are passed to the underlaying
+  //   encoder, frame encode complete callback is not intercepted.
   // * Multi-encoder simulcast or singlecast if layers are deactivated
-  //   (total_streams_count > 1 and active_streams_count >= 1). SEA creates
-  //   N=active_streams_count encoders and configures each to produce a single
-  //   stream.
+  //   (active_streams_count >= 1). SEA creates N=active_streams_count encoders
+  //   and configures each to produce a single stream.
 
+  int active_streams_count = CountActiveStreams(*inst);
+  // If we only have a single active layer it is better to create an encoder
+  // with only one configured layer than creating it with all-but-one disabled
+  // layers because that way we control scaling.
+  bool separate_encoders_needed =
+      !encoder_context->encoder().GetEncoderInfo().supports_simulcast ||
+      active_streams_count == 1;
   // Singlecast or simulcast with simulcast-capable underlaying encoder.
-  if (total_streams_count_ == 1 ||
-      encoder_context->encoder().GetEncoderInfo().supports_simulcast) {
+  if (total_streams_count_ == 1 || !separate_encoders_needed) {
     int ret = encoder_context->encoder().InitEncode(&codec_, settings);
     if (ret >= 0) {
-      int active_streams_count = CountActiveStreams(*inst);
       stream_contexts_.emplace_back(
           /*parent=*/nullptr, std::move(encoder_context),
           /*framerate_controller=*/nullptr, /*stream_idx=*/0, codec_.width,