diff --git a/api/video_codecs/libaom_av1_encoder_factory.cc b/api/video_codecs/libaom_av1_encoder_factory.cc
index eab6eaefe8..6cb789b8d5 100644
--- a/api/video_codecs/libaom_av1_encoder_factory.cc
+++ b/api/video_codecs/libaom_av1_encoder_factory.cc
@@ -766,7 +766,6 @@ void LibaomAv1Encoder::Encode(
         result.bitstream_data = EncodedImageBuffer::Create(
             static_cast<uint8_t*>(pkt->data.frame.buf), pkt->data.frame.sz);
         result.spatial_id = sid;
-        result.referenced_buffers = settings.reference_buffers;
         break;
       }
     }
diff --git a/api/video_codecs/simple_encoder_wrapper.cc b/api/video_codecs/simple_encoder_wrapper.cc
index bb2eda6afb..51adfd5170 100644
--- a/api/video_codecs/simple_encoder_wrapper.cc
+++ b/api/video_codecs/simple_encoder_wrapper.cc
@@ -200,7 +200,7 @@ void SimpleEncoderWrapper::Encode(
         res.frame_type = data->frame_type;
         res.bitstream_data = std::move(data->bitstream_data);
         res.generic_frame_info = infos[data->spatial_id];
-        if (data->referenced_buffers.empty()) {
+        if (res.frame_type == FrameType::kKeyframe) {
           // Keyframe
           res.dependency_structure = ds;
         }
diff --git a/api/video_codecs/video_encoder_interface.h b/api/video_codecs/video_encoder_interface.h
index 9d0ce970ef..4e0726436c 100644
--- a/api/video_codecs/video_encoder_interface.h
+++ b/api/video_codecs/video_encoder_interface.h
@@ -72,7 +72,6 @@ class VideoEncoderInterface {
     FrameType frame_type;
     int spatial_id;
     int encoded_qp;
-    std::vector<int> referenced_buffers;
   };
 
   using EncodeResult = std::variant<EncodingError, EncodedData>;