diff --git a/modules/video_coding/svc/scalability_structure_full_svc.cc b/modules/video_coding/svc/scalability_structure_full_svc.cc
index 9f66a5ad83..7177427c16 100644
--- a/modules/video_coding/svc/scalability_structure_full_svc.cc
+++ b/modules/video_coding/svc/scalability_structure_full_svc.cc
@@ -173,7 +173,6 @@ ScalabilityStructureFullSvc::NextFrameConfig(bool restart) {
           config.Update(BufferIndex(sid, /*tid=*/0));
         }
 
-        can_reference_t0_frame_for_spatial_id_.set(sid);
         spatial_dependency_buffer_id = BufferIndex(sid, /*tid=*/0);
       }
       break;
@@ -256,6 +255,9 @@ GenericFrameInfo ScalabilityStructureFullSvc::OnEncodeDone(
   // pattern defered here from the `NextFrameConfig`.
   // In particular creating VP9 references rely on this behavior.
   last_pattern_ = static_cast<FramePattern>(config.Id());
+  if (config.TemporalId() == 0) {
+    can_reference_t0_frame_for_spatial_id_.set(config.SpatialId());
+  }
   if (config.TemporalId() == 1) {
     can_reference_t1_frame_for_spatial_id_.set(config.SpatialId());
   }
diff --git a/modules/video_coding/svc/scalability_structure_full_svc_unittest.cc b/modules/video_coding/svc/scalability_structure_full_svc_unittest.cc
index 9ccbe21f75..1c0a8be8f1 100644
--- a/modules/video_coding/svc/scalability_structure_full_svc_unittest.cc
+++ b/modules/video_coding/svc/scalability_structure_full_svc_unittest.cc
@@ -21,6 +21,27 @@ namespace {
 using ::testing::IsEmpty;
 using ::testing::SizeIs;
 
+TEST(ScalabilityStructureL3T3Test, SkipT0FrameByEncoderKeepsReferencesValid) {
+  std::vector<GenericFrameInfo> frames;
+  ScalabilityStructureL3T3 structure;
+  ScalabilityStructureWrapper wrapper(structure);
+
+  // Only S0T0 decode target is enabled.
+  structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/1, /*s1=*/0));
+  // Encoder generates S0T0 key frame.
+  wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
+  EXPECT_THAT(frames, SizeIs(1));
+  // Spatial layers 1 is enabled.
+  structure.OnRatesUpdated(EnableTemporalLayers(/*s0=*/1, /*s1=*/1));
+  // Encoder tries to generate S0T0 and S1T0 delta frames but they are dropped.
+  structure.NextFrameConfig(/*restart=*/false);
+  // Encoder successfully generates S0T0 and S1T0 delta frames.
+  wrapper.GenerateFrames(/*num_temporal_units=*/1, frames);
+  EXPECT_THAT(frames, SizeIs(3));
+
+  EXPECT_TRUE(wrapper.FrameReferencesAreValid(frames));
+}
+
 TEST(ScalabilityStructureL3T3Test, SkipS1T1FrameKeepsStructureValid) {
   ScalabilityStructureL3T3 structure;
   ScalabilityStructureWrapper wrapper(structure);
diff --git a/test/fuzzers/BUILD.gn b/test/fuzzers/BUILD.gn
index 8300fca3a4..019b845727 100644
--- a/test/fuzzers/BUILD.gn
+++ b/test/fuzzers/BUILD.gn
@@ -648,6 +648,7 @@ if (rtc_build_libvpx) {
       "//third_party/abseil-cpp/absl/base:core_headers",
       "//third_party/abseil-cpp/absl/container:inlined_vector",
     ]
+    seed_corpus = "corpora/vp9-encoder-references-corpus"
     defines = [ "RTC_ENABLE_VP9" ]
   }
 }
diff --git a/test/fuzzers/corpora/vp9-encoder-references-corpus/0cee4d5fd2905dc1fb2979f10a9724265b7075e2 b/test/fuzzers/corpora/vp9-encoder-references-corpus/0cee4d5fd2905dc1fb2979f10a9724265b7075e2
new file mode 100644
index 0000000000..febe4ad130
Binary files /dev/null and b/test/fuzzers/corpora/vp9-encoder-references-corpus/0cee4d5fd2905dc1fb2979f10a9724265b7075e2 differ
diff --git a/test/fuzzers/corpora/vp9-encoder-references-corpus/a8b3fb7be82395c9462684c766841d668dc0029f b/test/fuzzers/corpora/vp9-encoder-references-corpus/a8b3fb7be82395c9462684c766841d668dc0029f
new file mode 100644
index 0000000000..1bd09373c8
Binary files /dev/null and b/test/fuzzers/corpora/vp9-encoder-references-corpus/a8b3fb7be82395c9462684c766841d668dc0029f differ