admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pass receive_time through frame transformer

Browse Source 
Bug: webrtc:344347965
Change-Id: Iee5ae13487f57f2b0c98dd6fb6a14286ff317fbb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/358100
Reviewed-by: Tony Herre <herre@google.com>
Commit-Queue: Lionel Koenig <lionelk@webrtc.org>
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42717}
This commit is contained in:
Lionel Koenig Gélas 2024-07-30 15:47:41 +02:00 committed by WebRTC LUCI CQ
parent 6f8c9afe34
commit b4462510c3
10 changed files with 99 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/BUILD.gn
View File

@ -1345,6 +1345,7 @@ if (rtc_include_tests) {
deps = [
":frame_transformer_interface",
"../api/units:timestamp",
"../test:test_support",
]
}

8
api/frame_transformer_factory.cc
View File

@ -13,6 +13,7 @@
#include <memory>
#include "api/frame_transformer_interface.h"
#include "audio/channel_receive_frame_transformer_delegate.h"
#include "audio/channel_send_frame_transformer_delegate.h"
#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
#include "rtc_base/checks.h"
@ -31,13 +32,16 @@ std::unique_ptr<TransformableVideoFrameInterface> CreateVideoReceiverFrame() {
std::unique_ptr<TransformableAudioFrameInterface> CloneAudioFrame(
TransformableAudioFrameInterface* original) {
// At the moment, only making sender frames is supported.
if (original->GetDirection() ==
TransformableAudioFrameInterface::Direction::kReceiver)
return CloneReceiverAudioFrame(original);
return CloneSenderAudioFrame(original);
}
std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame(
TransformableVideoFrameInterface* original) {
// At the moment, only making sender frames from receiver frames is supported.
// At the moment, only making sender frames from receiver frames is
// supported.
return CloneSenderVideoFrame(original);
}

4
api/frame_transformer_interface.h
View File

@ -105,6 +105,10 @@ class TransformableAudioFrameInterface : public TransformableFrameInterface {
// dBov. 127 represents digital silence. Only present on remote frames if
// the audio level header extension was included.
virtual absl::optional<uint8_t> AudioLevel() const = 0;
// Timestamp at which the packet has been first seen on the network interface.
// Only defined for received audio packet.
virtual absl::optional<Timestamp> ReceiveTime() const = 0;
};
// Objects implement this interface to be notified with the transformed frame.

3
api/test/mock_transformable_audio_frame.h
View File

@ -14,6 +14,7 @@
#include <string>
#include "api/frame_transformer_interface.h"
#include "api/units/timestamp.h"
#include "test/gmock.h"
namespace webrtc {
@ -50,6 +51,8 @@ class MockTransformableAudioFrame : public TransformableAudioFrameInterface {
(),
(const, override));
MOCK_METHOD(absl::optional<uint8_t>, AudioLevel, (), (const, override));
MOCK_METHOD(absl::optional<Timestamp>, ReceiveTime, (), (const, override));
};
} // namespace webrtc

2
audio/BUILD.gn
View File

@ -65,6 +65,7 @@ rtc_library("audio") {
"../api/task_queue:pending_task_safety_flag",
"../api/transport/rtp:rtp_source",
"../api/units:time_delta",
"../api/units:timestamp",
"../call:audio_sender_interface",
"../call:bitrate_allocator",
"../call:call_interfaces",
@ -162,6 +163,7 @@ if (rtc_include_tests) {
":audio_end_to_end_test",
":channel_receive_unittest",
"../api:array_view",
"../api:frame_transformer_factory",
"../api:frame_transformer_interface",
"../api:libjingle_peerconnection_api",
"../api:make_ref_counted",

10
audio/channel_receive.cc
View File

@ -25,6 +25,7 @@
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "audio/audio_level.h"
#include "audio/channel_receive_frame_transformer_delegate.h"
#include "audio/channel_send.h"
@ -371,11 +372,10 @@ void ChannelReceive::InitFrameTransformerDelegate(
// the delegate to receive transformed audio.
ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback
receive_audio_callback = [this](rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header) {
const RTPHeader& header,
Timestamp receive_time) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
// TODO(lionelk): Get the receive time.
OnReceivedPayloadData(packet, header,
/*receive_time=*/Timestamp::MinusInfinity());
OnReceivedPayloadData(packet, header, receive_time);
};
frame_transformer_delegate_ =
rtc::make_ref_counted<ChannelReceiveFrameTransformerDelegate>(
@ -723,7 +723,7 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet,
<< (it != payload_type_map_.end() ? it->second.name
: "x-unknown");
frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_,
mime_type.str());
mime_type.str(), receive_time);
} else {
OnReceivedPayloadData(payload_data, header, receive_time);
}

48
audio/channel_receive_frame_transformer_delegate.cc
View File

@ -22,7 +22,9 @@
#include "api/scoped_refptr.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/timestamp.h"
#include "rtc_base/buffer.h"
#include "rtc_base/string_encode.h"
namespace webrtc {
@ -32,12 +34,14 @@ class TransformableIncomingAudioFrame
TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload,
const RTPHeader& header,
uint32_t ssrc,
const std::string& codec_mime_type)
const std::string& codec_mime_type,
Timestamp receive_time)
: TransformableAudioFrameInterface(Passkey()),
payload_(payload.data(), payload.size()),
header_(header),
ssrc_(ssrc),
codec_mime_type_(codec_mime_type) {}
codec_mime_type_(codec_mime_type),
receive_time_(receive_time) {}
~TransformableIncomingAudioFrame() override = default;
rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
@ -86,11 +90,18 @@ class TransformableIncomingAudioFrame
return absl::nullopt;
}
absl::optional<Timestamp> ReceiveTime() const override {
return receive_time_ == Timestamp::MinusInfinity()
? absl::nullopt
: absl::optional<Timestamp>(receive_time_);
}
private:
rtc::Buffer payload_;
RTPHeader header_;
uint32_t ssrc_;
std::string codec_mime_type_;
Timestamp receive_time_;
};
ChannelReceiveFrameTransformerDelegate::ChannelReceiveFrameTransformerDelegate(
@ -118,14 +129,15 @@ void ChannelReceiveFrameTransformerDelegate::Transform(
rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
uint32_t ssrc,
const std::string& codec_mime_type) {
const std::string& codec_mime_type,
Timestamp receive_time) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
if (short_circuit_) {
receive_frame_callback_(packet, header);
receive_frame_callback_(packet, header, receive_time);
} else {
frame_transformer_->Transform(
std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
codec_mime_type));
std::make_unique<TransformableIncomingAudioFrame>(
packet, header, ssrc, codec_mime_type, receive_time));
}
}
@ -152,11 +164,13 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
if (!receive_frame_callback_)
return;
auto* transformed_frame =
static_cast<TransformableAudioFrameInterface*>(frame.get());
Timestamp receive_time =
transformed_frame->ReceiveTime().value_or(Timestamp::MinusInfinity());
RTPHeader header;
if (frame->GetDirection() ==
TransformableFrameInterface::Direction::kSender) {
auto* transformed_frame =
static_cast<TransformableAudioFrameInterface*>(frame.get());
header.payloadType = transformed_frame->GetPayloadType();
header.timestamp = transformed_frame->GetTimestamp();
header.ssrc = transformed_frame->GetSsrc();
@ -166,16 +180,16 @@ void ChannelReceiveFrameTransformerDelegate::ReceiveFrame(
transformed_frame->AbsoluteCaptureTimestamp().value();
}
} else {
auto* transformed_frame =
auto* transformed_incoming_frame =
static_cast<TransformableIncomingAudioFrame*>(frame.get());
header = transformed_frame->Header();
header = transformed_incoming_frame->Header();
}
// TODO(crbug.com/1464860): Take an explicit struct with the required
// information rather than the RTPHeader to make it easier to
// construct the required information when injecting transformed frames not
// originally from this receiver.
receive_frame_callback_(frame->GetData(), header);
receive_frame_callback_(frame->GetData(), header, receive_time);
}
rtc::scoped_refptr<FrameTransformerInterface>
@ -184,4 +198,16 @@ ChannelReceiveFrameTransformerDelegate::FrameTransformer() {
return frame_transformer_;
}
std::unique_ptr<TransformableAudioFrameInterface> CloneReceiverAudioFrame(
TransformableAudioFrameInterface* original) {
RTC_CHECK(original->GetDirection() ==
TransformableFrameInterface::Direction::kReceiver);
auto* original_incoming_frame =
static_cast<TransformableIncomingAudioFrame*>(original);
return std::make_unique<TransformableIncomingAudioFrame>(
original->GetData(), original_incoming_frame->Header(),
original->GetSsrc(), original->GetMimeType(),
original->ReceiveTime().value_or(Timestamp::MinusInfinity()));
}
} // namespace webrtc

10
audio/channel_receive_frame_transformer_delegate.h
View File

@ -18,6 +18,7 @@
#include "api/rtp_headers.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/timestamp.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/thread.h"
@ -30,7 +31,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
public:
using ReceiveFrameCallback =
std::function<void(rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header)>;
const RTPHeader& header,
Timestamp receive_time)>;
ChannelReceiveFrameTransformerDelegate(
ReceiveFrameCallback receive_frame_callback,
rtc::scoped_refptr<FrameTransformerInterface> frame_transformer,
@ -51,7 +53,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
void Transform(rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
uint32_t ssrc,
const std::string& codec_mime_type);
const std::string& codec_mime_type,
Timestamp receive_time);
// Implements TransformedFrameCallback. Can be called on any thread.
void OnTransformedFrame(
@ -78,5 +81,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
bool short_circuit_ RTC_GUARDED_BY(sequence_checker_) = false;
};
std::unique_ptr<TransformableAudioFrameInterface> CloneReceiverAudioFrame(
TransformableAudioFrameInterface* original);
} // namespace webrtc
#endif // AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_

45
audio/channel_receive_frame_transformer_delegate_unittest.cc
View File

@ -15,13 +15,14 @@
#include <utility>
#include "api/array_view.h"
#include "api/frame_transformer_factory.h"
#include "api/frame_transformer_interface.h"
#include "api/make_ref_counted.h"
#include "api/rtp_headers.h"
#include "api/scoped_refptr.h"
#include "api/test/mock_frame_transformer.h"
#include "api/test/mock_transformable_audio_frame.h"
#include "audio/channel_send_frame_transformer_delegate.h"
#include "api/units/timestamp.h"
#include "rtc_base/thread.h"
#include "test/gmock.h"
#include "test/gtest.h"
@ -34,15 +35,21 @@ using ::testing::ElementsAre;
using ::testing::NiceMock;
using ::testing::SaveArg;
constexpr Timestamp kFakeReceiveTimestamp = Timestamp::Millis(1234567);
class MockChannelReceive {
public:
MOCK_METHOD(void,
ReceiveFrame,
(rtc::ArrayView<const uint8_t> packet, const RTPHeader& header));
(rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
Timestamp receive_time));
ChannelReceiveFrameTransformerDelegate::ReceiveFrameCallback callback() {
return [this](rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header) { ReceiveFrame(packet, header); };
return [this](rtc::ArrayView<const uint8_t> packet, const RTPHeader& header,
Timestamp receive_time) {
ReceiveFrame(packet, header, receive_time);
};
}
};
@ -100,7 +107,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
[&callback](std::unique_ptr<TransformableFrameInterface> frame) {
callback->OnTransformedFrame(std::move(frame));
});
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
kFakeReceiveTimestamp);
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
@ -125,15 +133,17 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
const uint8_t data[] = {1, 2, 3, 4};
rtc::ArrayView<const uint8_t> packet(data, sizeof(data));
RTPHeader header;
EXPECT_CALL(mock_channel, ReceiveFrame(ElementsAre(1, 2, 3, 4), _));
EXPECT_CALL(mock_channel,
ReceiveFrame(ElementsAre(1, 2, 3, 4), _, kFakeReceiveTimestamp));
ON_CALL(*mock_frame_transformer, Transform)
.WillByDefault([&callback](
std::unique_ptr<TransformableFrameInterface> frame) {
auto* transformed_frame =
static_cast<TransformableAudioFrameInterface*>(frame.get());
callback->OnTransformedFrame(CloneSenderAudioFrame(transformed_frame));
});
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
.WillByDefault(
[&callback](std::unique_ptr<TransformableFrameInterface> frame) {
auto* transformed_frame =
static_cast<TransformableAudioFrameInterface*>(frame.get());
callback->OnTransformedFrame(CloneAudioFrame(transformed_frame));
});
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
kFakeReceiveTimestamp);
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
@ -178,7 +188,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
EXPECT_CALL(*mock_frame_transformer, Transform).Times(0);
// Will pass the frame straight to the channel.
EXPECT_CALL(mock_channel, ReceiveFrame);
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
kFakeReceiveTimestamp);
}
TEST(ChannelReceiveFrameTransformerDelegateTest,
@ -205,7 +216,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
[&](std::unique_ptr<TransformableFrameInterface> transform_frame) {
frame = std::move(transform_frame);
});
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
kFakeReceiveTimestamp);
EXPECT_TRUE(frame);
auto* audio_frame =
@ -242,7 +254,8 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
[&](std::unique_ptr<TransformableFrameInterface> transform_frame) {
frame = std::move(transform_frame);
});
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus",
kFakeReceiveTimestamp);
EXPECT_TRUE(frame);
auto* audio_frame =

4
audio/channel_send_frame_transformer_delegate.cc
View File

@ -109,6 +109,10 @@ class TransformableOutgoingAudioFrame
return audio_level_dbov_;
}
absl::optional<Timestamp> ReceiveTime() const override {
return absl::nullopt;
}
private:
AudioFrameType frame_type_;
uint8_t payload_type_;