diff --git a/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc
index ee61893563..30d5287c94 100644
--- a/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc
+++ b/api/video_codecs/test/video_decoder_software_fallback_wrapper_unittest.cc
@@ -218,6 +218,68 @@ TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
   fallback_wrapper_->Release();
 }
 
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest, FallbacksOnTooManyErrors) {
+  VideoCodec codec = {};
+  fallback_wrapper_->InitDecode(&codec, 2);
+
+  fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
+  EncodedImage encoded_image;
+  encoded_image._frameType = VideoFrameType::kVideoFrameKey;
+  // Doesn't fallback from a single error.
+  fallback_wrapper_->Decode(encoded_image, false, -1);
+  EXPECT_STREQ("fake-decoder", fallback_wrapper_->ImplementationName());
+
+  // However, many frames with the same error, fallback should happen.
+  const int kNumFramesToEncode = 10;
+  for (int i = 0; i < kNumFramesToEncode; ++i) {
+    fallback_wrapper_->Decode(encoded_image, false, -1);
+  }
+  // Hard coded expected value since libvpx is the software implementation name
+  // for VP8. Change accordingly if the underlying implementation does.
+  EXPECT_STREQ("libvpx (fallback from: fake-decoder)",
+               fallback_wrapper_->ImplementationName());
+  fallback_wrapper_->Release();
+}
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
+       DoesNotFallbackOnDeltaFramesErrors) {
+  VideoCodec codec = {};
+  fallback_wrapper_->InitDecode(&codec, 2);
+
+  fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
+  EncodedImage encoded_image;
+  encoded_image._frameType = VideoFrameType::kVideoFrameDelta;
+
+  // Many decoded frames with the same error
+  const int kNumFramesToEncode = 10;
+  for (int i = 0; i < kNumFramesToEncode; ++i) {
+    fallback_wrapper_->Decode(encoded_image, false, -1);
+  }
+  EXPECT_STREQ("fake-decoder", fallback_wrapper_->ImplementationName());
+
+  fallback_wrapper_->Release();
+}
+
+TEST_F(VideoDecoderSoftwareFallbackWrapperTest,
+       DoesNotFallbacksOnNonConsequtiveErrors) {
+  VideoCodec codec = {};
+  fallback_wrapper_->InitDecode(&codec, 2);
+
+  EncodedImage encoded_image;
+  encoded_image._frameType = VideoFrameType::kVideoFrameKey;
+
+  const int kNumFramesToEncode = 10;
+  for (int i = 0; i < kNumFramesToEncode; ++i) {
+    // Interleaved errors and successful decodes.
+    fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR;
+    fallback_wrapper_->Decode(encoded_image, false, -1);
+    fake_decoder_->decode_return_code_ = WEBRTC_VIDEO_CODEC_OK;
+    fallback_wrapper_->Decode(encoded_image, false, -1);
+  }
+  EXPECT_STREQ("fake-decoder", fallback_wrapper_->ImplementationName());
+  fallback_wrapper_->Release();
+}
+
 class ForcedSoftwareDecoderFallbackTest
     : public VideoDecoderSoftwareFallbackWrapperTest {
  public:
diff --git a/api/video_codecs/video_decoder_software_fallback_wrapper.cc b/api/video_codecs/video_decoder_software_fallback_wrapper.cc
index f78d9b885f..128087f207 100644
--- a/api/video_codecs/video_decoder_software_fallback_wrapper.cc
+++ b/api/video_codecs/video_decoder_software_fallback_wrapper.cc
@@ -30,6 +30,8 @@ namespace webrtc {
 
 namespace {
 
+constexpr size_t kMaxConsequtiveHwErrors = 4;
+
 class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder {
  public:
   VideoDecoderSoftwareFallbackWrapper(
@@ -74,6 +76,7 @@ class VideoDecoderSoftwareFallbackWrapper final : public VideoDecoder {
   const std::string fallback_implementation_name_;
   DecodedImageCallback* callback_;
   int32_t hw_decoded_frames_since_last_fallback_;
+  size_t hw_consequtive_generic_errors_;
 };
 
 VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
@@ -86,7 +89,8 @@ VideoDecoderSoftwareFallbackWrapper::VideoDecoderSoftwareFallbackWrapper(
           std::string(fallback_decoder_->ImplementationName()) +
           " (fallback from: " + hw_decoder_->ImplementationName() + ")"),
       callback_(nullptr),
-      hw_decoded_frames_since_last_fallback_(0) {}
+      hw_decoded_frames_since_last_fallback_(0),
+      hw_consequtive_generic_errors_(0) {}
 VideoDecoderSoftwareFallbackWrapper::~VideoDecoderSoftwareFallbackWrapper() =
     default;
 
@@ -196,14 +200,24 @@ int32_t VideoDecoderSoftwareFallbackWrapper::Decode(
       int32_t ret = WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
       ret = hw_decoder_->Decode(input_image, missing_frames, render_time_ms);
       if (ret != WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) {
-        if (ret == WEBRTC_VIDEO_CODEC_OK) {
+        if (ret != WEBRTC_VIDEO_CODEC_ERROR) {
           ++hw_decoded_frames_since_last_fallback_;
+          hw_consequtive_generic_errors_ = 0;
+          return ret;
+        }
+        if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
+          // Only count errors on key-frames, since generic errors can happen
+          // with hw decoder due to many arbitrary reasons.
+          // However, requesting a key-frame is supposed to fix the issue.
+          ++hw_consequtive_generic_errors_;
+        }
+        if (hw_consequtive_generic_errors_ < kMaxConsequtiveHwErrors) {
+          return ret;
         }
-        return ret;
       }
 
       // HW decoder returned WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE or
-      // initialization failed, fallback to software.
+      // too many generic errors on key-frames encountered.
       if (!InitFallbackDecoder()) {
         return ret;
       }