From ee5b6de5aa2bf48c0dd27d6a7a2cb3a03b359f12 Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Mon, 22 Jun 2020 17:53:01 +0200 Subject: [PATCH] Add helper for DependencyDescriptor rtp header extension to decide when to set active_decode_target_bitmask field Bug: webrtc:10342 Change-Id: I348d7467a72b45651455f4574fe8fda3c77ebbae Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177400 Reviewed-by: Philip Eliasson Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#31555} --- modules/rtp_rtcp/BUILD.gn | 3 + .../source/active_decode_targets_helper.cc | 127 ++++++++ .../source/active_decode_targets_helper.h | 60 ++++ .../active_decode_targets_helper_unittest.cc | 295 ++++++++++++++++++ 4 files changed, 485 insertions(+) create mode 100644 modules/rtp_rtcp/source/active_decode_targets_helper.cc create mode 100644 modules/rtp_rtcp/source/active_decode_targets_helper.h create mode 100644 modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 5a01e131bd..0446799fb7 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -140,6 +140,8 @@ rtc_library("rtp_rtcp") { "source/absolute_capture_time_receiver.h", "source/absolute_capture_time_sender.cc", "source/absolute_capture_time_sender.h", + "source/active_decode_targets_helper.cc", + "source/active_decode_targets_helper.h", "source/create_video_rtp_depacketizer.cc", "source/create_video_rtp_depacketizer.h", "source/deprecated/deprecated_rtp_sender_egress.cc", @@ -438,6 +440,7 @@ if (rtc_include_tests) { sources = [ "source/absolute_capture_time_receiver_unittest.cc", "source/absolute_capture_time_sender_unittest.cc", + "source/active_decode_targets_helper_unittest.cc", "source/byte_io_unittest.cc", "source/fec_private_tables_bursty_unittest.cc", "source/flexfec_header_reader_writer_unittest.cc", diff --git a/modules/rtp_rtcp/source/active_decode_targets_helper.cc b/modules/rtp_rtcp/source/active_decode_targets_helper.cc new file mode 100644 index 0000000000..a14426e144 --- /dev/null +++ b/modules/rtp_rtcp/source/active_decode_targets_helper.cc @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/active_decode_targets_helper.h" + +#include + +#include "api/array_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +// Returns mask of ids of chains previous frame is part of. +// Assumes for each chain frames are seen in order and no frame on any chain is +// missing. That assumptions allows a simple detection when previous frame is +// part of a chain. +std::bitset<32> LastSendOnChain(int frame_diff, + rtc::ArrayView chain_diffs) { + std::bitset<32> bitmask = 0; + for (size_t i = 0; i < chain_diffs.size(); ++i) { + if (frame_diff == chain_diffs[i]) { + bitmask.set(i); + } + } + return bitmask; +} + +// Returns bitmask with first `num` bits set to 1. +std::bitset<32> AllActive(size_t num) { + RTC_DCHECK_LE(num, 32); + return (~uint32_t{0}) >> (32 - num); +} + +// Returns bitmask of chains that protect at least one active decode target. +std::bitset<32> ActiveChains( + rtc::ArrayView decode_target_protected_by_chain, + int num_chains, + std::bitset<32> active_decode_targets) { + std::bitset<32> active_chains = 0; + for (size_t dt = 0; dt < decode_target_protected_by_chain.size(); ++dt) { + if (dt < active_decode_targets.size() && !active_decode_targets[dt]) { + continue; + } + // chain_idx == num_chains is valid and means the decode target is + // not protected by any chain. + int chain_idx = decode_target_protected_by_chain[dt]; + if (chain_idx < num_chains) { + active_chains.set(chain_idx); + } + } + return active_chains; +} + +} // namespace + +void ActiveDecodeTargetsHelper::OnFrame( + rtc::ArrayView decode_target_protected_by_chain, + std::bitset<32> active_decode_targets, + bool is_keyframe, + int64_t frame_id, + rtc::ArrayView chain_diffs) { + const int num_chains = chain_diffs.size(); + if (num_chains == 0) { + // Avoid printing the warning + // when already printed the warning for the same active decode targets, or + // when active_decode_targets are not changed from it's default value of + // all are active, including non-existent decode targets. + if (last_active_decode_targets_ != active_decode_targets && + !active_decode_targets.all()) { + RTC_LOG(LS_WARNING) << "No chains are configured, but some decode " + "targets might be inactive. Unsupported."; + } + last_active_decode_targets_ = active_decode_targets; + return; + } + const size_t num_decode_targets = decode_target_protected_by_chain.size(); + RTC_DCHECK_GT(num_decode_targets, 0); + std::bitset<32> all_decode_targets = AllActive(num_decode_targets); + // Default value for active_decode_targets is 'all are active', i.e. all bits + // are set. Default value is set before number of decode targets is known. + // It is up to this helper to make the value cleaner and unset unused bits. + active_decode_targets &= all_decode_targets; + + if (is_keyframe) { + // Key frame resets the state. + last_active_decode_targets_ = all_decode_targets; + unsent_on_chain_.reset(); + } else { + // Update state assuming previous frame was sent. + unsent_on_chain_ &= + ~LastSendOnChain(frame_id - last_frame_id_, chain_diffs); + } + // Save for the next call to OnFrame. + // Though usually `frame_id == last_frame_id_ + 1`, it might not be so when + // frame id space is shared by several simulcast rtp streams. + last_frame_id_ = frame_id; + + if (active_decode_targets == last_active_decode_targets_) { + return; + } + last_active_decode_targets_ = active_decode_targets; + + // Frames that are part of inactive chains might not be produced by the + // encoder. Thus stop sending `active_decode_target` bitmask when it is sent + // on all active chains rather than on all chains. + unsent_on_chain_ = ActiveChains(decode_target_protected_by_chain, num_chains, + active_decode_targets); + if (unsent_on_chain_.none()) { + // Active decode targets are not protected by any chains. To be on the + // safe side always send the active_decode_targets_bitmask from now on. + RTC_LOG(LS_WARNING) + << "Active decode targets protected by no chains. (In)active decode " + "targets information will be send overreliably."; + unsent_on_chain_.set(1); + } +} + +} // namespace webrtc diff --git a/modules/rtp_rtcp/source/active_decode_targets_helper.h b/modules/rtp_rtcp/source/active_decode_targets_helper.h new file mode 100644 index 0000000000..b51144d9cb --- /dev/null +++ b/modules/rtp_rtcp/source/active_decode_targets_helper.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_ACTIVE_DECODE_TARGETS_HELPER_H_ +#define MODULES_RTP_RTCP_SOURCE_ACTIVE_DECODE_TARGETS_HELPER_H_ + +#include + +#include + +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// Helper class that decides when active_decode_target_bitmask should be written +// into the dependency descriptor rtp header extension. +// See: https://aomediacodec.github.io/av1-rtp-spec/#a44-switching +// This class is thread-compatible +class ActiveDecodeTargetsHelper { + public: + ActiveDecodeTargetsHelper() = default; + ActiveDecodeTargetsHelper(const ActiveDecodeTargetsHelper&) = delete; + ActiveDecodeTargetsHelper& operator=(const ActiveDecodeTargetsHelper&) = + delete; + ~ActiveDecodeTargetsHelper() = default; + + // Decides if active decode target bitmask should be attached to the frame + // that is about to be sent. + void OnFrame(rtc::ArrayView decode_target_protected_by_chain, + std::bitset<32> active_decode_targets, + bool is_keyframe, + int64_t frame_id, + rtc::ArrayView chain_diffs); + + // Returns active decode target to attach to the dependency descriptor. + absl::optional ActiveDecodeTargetsBitmask() const { + if (unsent_on_chain_.none()) + return absl::nullopt; + return last_active_decode_targets_.to_ulong(); + } + + private: + // `unsent_on_chain_[i]` indicates last active decode + // target bitmask wasn't attached to a packet on the chain with id `i`. + std::bitset<32> unsent_on_chain_ = 0; + std::bitset<32> last_active_decode_targets_ = 0; + int64_t last_frame_id_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_ACTIVE_DECODE_TARGETS_HELPER_H_ diff --git a/modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc b/modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc new file mode 100644 index 0000000000..651ab22e54 --- /dev/null +++ b/modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/active_decode_targets_helper.h" + +#include + +#include "absl/types/optional.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +constexpr std::bitset<32> kAll = ~uint32_t{0}; +} // namespace + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptOnKeyFrameWhenAllDecodeTargetsAreActive) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptOnKeyFrameWhenAllDecodeTargetsAreActiveAfterDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/3, chain_diffs_key); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsBitmaskOnKeyFrameWhenSomeDecodeTargetsAreInactive) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsBitmaskOnKeyFrameWhenSomeDecodeTargetsAreInactiveAfterDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/3, chain_diffs_key); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptWhenActiveDecodeTargetsAreUnused) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptOnDeltaFrameAfterSentOnKeyFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsNewBitmaskOnDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsBitmaskWhenAllDecodeTargetsReactivatedOnDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + ASSERT_NE(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + // Reactive all the decode targets + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/false, /*frame_id=*/3, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b11u); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsNulloptAfterSentOnAllActiveChains) { + // Active decode targets (0 and 1) are protected by chains 1 and 2. + const std::bitset<32> kSome = 0b011; + constexpr int kDecodeTargetProtectedByChain[] = {2, 1, 0}; + + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0, 0, 0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b111, + /*is_keyframe=*/true, + /*frame_id=*/0, chain_diffs_key); + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + int chain_diffs_delta1[] = {1, 1, 1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kSome, + /*is_keyframe=*/false, + /*frame_id=*/1, chain_diffs_delta1); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b011u); + + int chain_diffs_delta2[] = {2, 2, 1}; // Previous frame was part of chain#2 + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kSome, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta2); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b011u); + + // active_decode_targets_bitmask was send on chains 1 and 2. It was never sent + // on chain 0, but chain 0 only protects inactive decode target#2 + int chain_diffs_delta3[] = {3, 1, 2}; // Previous frame was part of chain#1 + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kSome, + /*is_keyframe=*/false, + /*frame_id=*/3, chain_diffs_delta3); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsBitmaskWhenChanged) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 1, 1}; + + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0, 0}; + helper.OnFrame(kDecodeTargetProtectedByChain, /*active_decode_targets=*/0b111, + /*is_keyframe=*/true, + /*frame_id=*/0, chain_diffs_key); + int chain_diffs_delta1[] = {1, 1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b011, + /*is_keyframe=*/false, + /*frame_id=*/1, chain_diffs_delta1); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b011u); + + int chain_diffs_delta2[] = {1, 2}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b101, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta2); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b101u); + + // active_decode_target_bitmask was send on chain0, but it was an old one. + int chain_diffs_delta3[] = {2, 1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b101, + /*is_keyframe=*/false, + /*frame_id=*/3, chain_diffs_delta3); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b101u); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsNulloptWhenChainsAreNotUsed) { + const rtc::ArrayView kDecodeTargetProtectedByChain; + const rtc::ArrayView kNoChainDiffs; + + ActiveDecodeTargetsHelper helper; + helper.OnFrame(kDecodeTargetProtectedByChain, /*active_decode_targets=*/kAll, + /*is_keyframe=*/true, + /*frame_id=*/0, kNoChainDiffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b101, + /*is_keyframe=*/false, + /*frame_id=*/1, kNoChainDiffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + KeepReturningBitmaskWhenAllChainsAreInactive) { + // Two decode targets, but single chain. + // 2nd decode target is not protected by any chain. + constexpr int kDecodeTargetProtectedByChain[] = {0, 1}; + + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, /*active_decode_targets=*/0b10, + /*is_keyframe=*/true, + /*frame_id=*/0, chain_diffs_key); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b10u); + + // Even though previous frame is part of the only chain, that inactive chain + // doesn't provide guaranted delivery. + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b10, + /*is_keyframe=*/false, + /*frame_id=*/1, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b10u); +} + +TEST(ActiveDecodeTargetsHelperTest, Supports32DecodeTargets) { + std::bitset<32> some; + std::vector decode_target_protected_by_chain(32); + for (int i = 0; i < 32; ++i) { + decode_target_protected_by_chain[i] = i; + some[i] = i % 2 == 0; + } + + ActiveDecodeTargetsHelper helper; + std::vector chain_diffs_key(32, 0); + helper.OnFrame(decode_target_protected_by_chain, + /*active_decode_targets=*/some, + /*is_keyframe=*/true, + /*frame_id=*/1, chain_diffs_key); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), some.to_ulong()); + std::vector chain_diffs_delta(32, 1); + helper.OnFrame(decode_target_protected_by_chain, + /*active_decode_targets=*/some, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + helper.OnFrame(decode_target_protected_by_chain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), kAll.to_ulong()); +} + +} // namespace webrtc