From 1e2d951762b735fa007331c6036f6ce25ad439a6 Mon Sep 17 00:00:00 2001 From: Tove Petersson Date: Sun, 5 Mar 2023 11:03:06 +0000 Subject: [PATCH] Add a clone method to the audio frame transformer API. This will clone an encoded audio frame into a sender frame. Bug: webrtc:14949 Change-Id: Ie62d9f5ec457541b335bde8f2f6e9b6d24704cf6 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/294560 Commit-Queue: Tove Petersson Reviewed-by: Harald Alvestrand Cr-Commit-Position: refs/heads/main@{#39480} --- api/BUILD.gn | 13 +++++++ api/frame_transformer_factory.cc | 7 ++++ api/frame_transformer_factory.h | 2 ++ api/test/mock_transformable_audio_frame.h | 35 +++++++++++++++++++ ...channel_send_frame_transformer_delegate.cc | 15 ++++++++ .../channel_send_frame_transformer_delegate.h | 4 +++ modules/rtp_rtcp/BUILD.gn | 1 + .../frame_transformer_factory_unittest.cc | 19 +++++++++- 8 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 api/test/mock_transformable_audio_frame.h diff --git a/api/BUILD.gn b/api/BUILD.gn index 0a5127e0bb..a9b816d298 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -1293,6 +1293,17 @@ if (rtc_include_tests) { ] } + rtc_source_set("mock_transformable_audio_frame") { + visibility = [ "*" ] + testonly = true + sources = [ "test/mock_transformable_audio_frame.h" ] + + deps = [ + ":frame_transformer_interface", + "../test:test_support", + ] + } + rtc_source_set("mock_transformable_video_frame") { visibility = [ "*" ] testonly = true @@ -1478,6 +1489,7 @@ if (rtc_include_tests) { ":mock_peerconnectioninterface", ":mock_rtp", ":mock_session_description_interface", + ":mock_transformable_audio_frame", ":mock_transformable_video_frame", ":mock_video_bitrate_allocator", ":mock_video_bitrate_allocator_factory", @@ -1548,6 +1560,7 @@ rtc_library("frame_transformer_factory") { deps = [ ":frame_transformer_interface", ":scoped_refptr", + "../audio:audio", "../modules/rtp_rtcp", "../rtc_base:refcount", "video:encoded_frame", diff --git a/api/frame_transformer_factory.cc b/api/frame_transformer_factory.cc index af08372e37..262ac0795c 100644 --- a/api/frame_transformer_factory.cc +++ b/api/frame_transformer_factory.cc @@ -10,6 +10,7 @@ #include "api/frame_transformer_factory.h" +#include "audio/channel_send_frame_transformer_delegate.h" #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" namespace webrtc { @@ -24,6 +25,12 @@ std::unique_ptr CreateVideoReceiverFrame() { return nullptr; } +std::unique_ptr CloneAudioFrame( + TransformableAudioFrameInterface* original) { + // At the moment, only making sender frames is supported. + return CloneSenderAudioFrame(original); +} + std::unique_ptr CloneVideoFrame( TransformableVideoFrameInterface* original) { // At the moment, only making sender frames from receiver frames is supported. diff --git a/api/frame_transformer_factory.h b/api/frame_transformer_factory.h index 8ba9c292d5..e0a7bdf4af 100644 --- a/api/frame_transformer_factory.h +++ b/api/frame_transformer_factory.h @@ -32,6 +32,8 @@ std::unique_ptr CreateVideoSenderFrame(); std::unique_ptr CreateVideoReceiverFrame(); // Creates a new frame with the same metadata as the original. // The original can be a sender or receiver frame. +RTC_EXPORT std::unique_ptr CloneAudioFrame( + TransformableAudioFrameInterface* original); RTC_EXPORT std::unique_ptr CloneVideoFrame( TransformableVideoFrameInterface* original); } // namespace webrtc diff --git a/api/test/mock_transformable_audio_frame.h b/api/test/mock_transformable_audio_frame.h new file mode 100644 index 0000000000..7a31ecd5a0 --- /dev/null +++ b/api/test/mock_transformable_audio_frame.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_MOCK_TRANSFORMABLE_AUDIO_FRAME_H_ +#define API_TEST_MOCK_TRANSFORMABLE_AUDIO_FRAME_H_ + +#include "api/frame_transformer_interface.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockTransformableAudioFrame : public TransformableAudioFrameInterface { + public: + MOCK_METHOD(rtc::ArrayView, GetData, (), (const, override)); + MOCK_METHOD(void, SetData, (rtc::ArrayView), (override)); + MOCK_METHOD(uint8_t, GetPayloadType, (), (const, override)); + MOCK_METHOD(uint32_t, GetSsrc, (), (const, override)); + MOCK_METHOD(uint32_t, GetTimestamp, (), (const, override)); + MOCK_METHOD(RTPHeader&, GetHeader, (), (const override)); + MOCK_METHOD(rtc::ArrayView, + GetContributingSources, + (), + (const override)); +}; + +} // namespace webrtc + +#endif // API_TEST_MOCK_TRANSFORMABLE_AUDIO_FRAME_H_ diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index 29bb0b81d8..bf7b49527c 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -127,4 +127,19 @@ void ChannelSendFrameTransformerDelegate::SendFrame( transformed_frame->GetAbsoluteCaptureTimestampMs()); } +std::unique_ptr CloneSenderAudioFrame( + TransformableAudioFrameInterface* original) { + AudioFrameType audio_frame_type = + original->GetHeader().extension.voiceActivity + ? AudioFrameType::kAudioFrameSpeech + : AudioFrameType::kAudioFrameCN; + + // TODO(crbug.com/webrtc/14949): Ensure the correct timestamps are passed. + return std::make_unique( + audio_frame_type, original->GetPayloadType(), original->GetTimestamp(), + /*rtp_start_timestamp=*/0u, original->GetData().data(), + original->GetData().size(), original->GetTimestamp(), + original->GetSsrc()); +} + } // namespace webrtc diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h index 6d9f0a8613..6bd0af70d7 100644 --- a/audio/channel_send_frame_transformer_delegate.h +++ b/audio/channel_send_frame_transformer_delegate.h @@ -77,5 +77,9 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { rtc::scoped_refptr frame_transformer_; rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_); }; + +std::unique_ptr CloneSenderAudioFrame( + TransformableAudioFrameInterface* original); + } // namespace webrtc #endif // AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_ diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 1c5510fb30..6b599fc384 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -709,6 +709,7 @@ if (rtc_include_tests) { sources = [ "source/frame_transformer_factory_unittest.cc" ] deps = [ "../../api:frame_transformer_factory", + "../../api:mock_transformable_audio_frame", "../../api:mock_transformable_video_frame", "../../api:transport_api", "../../call:video_stream_api", diff --git a/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc index 4c25330534..4ecc5a379f 100644 --- a/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc +++ b/modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc @@ -17,6 +17,7 @@ #include "absl/memory/memory.h" #include "api/call/transport.h" +#include "api/test/mock_transformable_audio_frame.h" #include "api/test/mock_transformable_video_frame.h" #include "call/video_receive_stream.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" @@ -28,8 +29,24 @@ namespace webrtc { namespace { +using testing::Each; +using testing::ElementsAreArray; using testing::NiceMock; using testing::Return; +using testing::ReturnRef; + +TEST(FrameTransformerFactory, CloneAudioFrame) { + NiceMock original_frame; + uint8_t data[10]; + std::fill_n(data, 10, 5); + rtc::ArrayView data_view(data); + ON_CALL(original_frame, GetData()).WillByDefault(Return(data_view)); + RTPHeader rtp_header; + ON_CALL(original_frame, GetHeader()).WillByDefault(ReturnRef(rtp_header)); + auto cloned_frame = CloneAudioFrame(&original_frame); + + EXPECT_THAT(cloned_frame->GetData(), ElementsAreArray(data)); +} TEST(FrameTransformerFactory, CloneVideoFrame) { NiceMock original_frame; @@ -46,7 +63,7 @@ TEST(FrameTransformerFactory, CloneVideoFrame) { auto cloned_frame = CloneVideoFrame(&original_frame); EXPECT_EQ(cloned_frame->GetData().size(), 10u); - EXPECT_THAT(cloned_frame->GetData(), testing::Each(5u)); + EXPECT_THAT(cloned_frame->GetData(), Each(5u)); EXPECT_EQ(cloned_frame->Metadata().GetCsrcs(), csrcs); }