From 04f4d126f84dbbb915f52d91807cabdabf08d483 Mon Sep 17 00:00:00 2001 From: ilnik Date: Mon, 19 Jun 2017 07:18:55 -0700 Subject: [PATCH] Implement timing frames. Timing information is gathered in EncodedImage, starting at encoders. Then it's sent using RTP header extension. In the end, it's gathered at the GenericDecoder. Actual reporting and tests will be in the next CLs. BUG=webrtc:7594 Review-Url: https://codereview.webrtc.org/2911193002 Cr-Commit-Position: refs/heads/master@{#18659} --- webrtc/api/BUILD.gn | 1 + webrtc/api/video/video_timing.h | 50 ++++++ webrtc/common_types.cc | 4 +- webrtc/common_types.h | 17 ++ webrtc/common_video/include/video_frame.h | 19 ++ webrtc/config.cc | 7 +- webrtc/config.h | 4 + webrtc/media/engine/webrtcvideoengine.cc | 2 + webrtc/modules/include/module_common_types.h | 2 + .../rtp_rtcp/include/rtp_rtcp_defines.h | 1 + .../source/rtp_header_extension_map.cc | 1 + .../rtp_rtcp/source/rtp_header_extensions.cc | 66 +++++++ .../rtp_rtcp/source/rtp_header_extensions.h | 19 ++ webrtc/modules/rtp_rtcp/source/rtp_packet.cc | 2 + .../rtp_rtcp/source/rtp_packet_to_send.h | 27 +++ .../rtp_rtcp/source/rtp_receiver_video.cc | 7 + .../modules/rtp_rtcp/source/rtp_rtcp_impl.cc | 2 + .../rtp_rtcp/source/rtp_rtcp_impl_unittest.cc | 1 + webrtc/modules/rtp_rtcp/source/rtp_sender.cc | 5 + .../rtp_rtcp/source/rtp_sender_unittest.cc | 75 ++++++++ .../rtp_rtcp/source/rtp_sender_video.cc | 11 ++ webrtc/modules/rtp_rtcp/source/rtp_utility.cc | 15 +- webrtc/modules/video_coding/BUILD.gn | 1 + webrtc/modules/video_coding/codec_database.cc | 9 + .../codecs/h264/h264_encoder_impl.cc | 3 +- .../codecs/vp8/simulcast_unittest.h | 3 + .../video_coding/codecs/vp8/vp8_impl.cc | 1 + .../video_coding/codecs/vp9/vp9_impl.cc | 3 +- webrtc/modules/video_coding/encoded_frame.cc | 1 + webrtc/modules/video_coding/encoded_frame.h | 4 + webrtc/modules/video_coding/frame_buffer.cc | 21 +++ webrtc/modules/video_coding/frame_object.cc | 28 +++ .../modules/video_coding/generic_decoder.cc | 30 +++- webrtc/modules/video_coding/generic_decoder.h | 2 + .../modules/video_coding/generic_encoder.cc | 118 +++++++++++- webrtc/modules/video_coding/generic_encoder.h | 31 ++++ .../video_coding/generic_encoder_unittest.cc | 168 ++++++++++++++++++ .../include/video_coding_defines.h | 9 +- webrtc/modules/video_coding/packet.cc | 3 +- webrtc/modules/video_coding/packet.h | 2 + .../video_coding/video_codec_initializer.cc | 5 +- .../src/jni/androidmediaencoder_jni.cc | 1 + .../Framework/Classes/VideoToolbox/encoder.mm | 2 +- webrtc/test/constants.cc | 1 + webrtc/test/constants.h | 1 + webrtc/test/fake_encoder.cc | 1 + webrtc/test/frame_generator_capturer.cc | 2 +- webrtc/test/fuzzers/rtp_packet_fuzzer.cc | 4 + webrtc/video/payload_router.cc | 15 ++ webrtc/video/rtp_video_stream_receiver.cc | 8 +- webrtc/video/video_quality_test.cc | 2 + webrtc/video/video_send_stream_tests.cc | 34 ++++ webrtc/video/vie_encoder.cc | 9 +- 53 files changed, 844 insertions(+), 16 deletions(-) create mode 100644 webrtc/api/video/video_timing.h create mode 100644 webrtc/modules/video_coding/generic_encoder_unittest.cc diff --git a/webrtc/api/BUILD.gn b/webrtc/api/BUILD.gn index d47483b81e..40741bced5 100644 --- a/webrtc/api/BUILD.gn +++ b/webrtc/api/BUILD.gn @@ -173,6 +173,7 @@ rtc_source_set("video_frame_api") { "video/video_frame_buffer.cc", "video/video_frame_buffer.h", "video/video_rotation.h", + "video/video_timing.h", ] deps = [ diff --git a/webrtc/api/video/video_timing.h b/webrtc/api/video/video_timing.h new file mode 100644 index 0000000000..a44a8ef68d --- /dev/null +++ b/webrtc/api/video/video_timing.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_API_VIDEO_VIDEO_TIMING_H_ +#define WEBRTC_API_VIDEO_VIDEO_TIMING_H_ + +#include +#include +#include "webrtc/base/checks.h" +#include "webrtc/base/safe_conversions.h" + +namespace webrtc { + +// Video timing timstamps in ms counted from capture_time_ms of a frame. +struct VideoTiming { + static const uint8_t kEncodeStartDeltaIdx = 0; + static const uint8_t kEncodeFinishDeltaIdx = 1; + static const uint8_t kPacketizationFinishDeltaIdx = 2; + static const uint8_t kPacerExitDeltaIdx = 3; + static const uint8_t kNetworkTimestampDeltaIdx = 4; + static const uint8_t kNetwork2TimestampDeltaIdx = 5; + + // Returns |time_ms - base_ms| capped at max 16-bit value. + // Used to fill this data structure as per + // https://webrtc.org/experiments/rtp-hdrext/video-timing/ extension stores + // 16-bit deltas of timestamps from packet capture time. + static uint16_t GetDeltaCappedMs(int64_t base_ms, int64_t time_ms) { + RTC_DCHECK_GE(time_ms, base_ms); + return rtc::saturated_cast(time_ms - base_ms); + } + + uint16_t encode_start_delta_ms; + uint16_t encode_finish_delta_ms; + uint16_t packetization_finish_delta_ms; + uint16_t pacer_exit_delta_ms; + uint16_t network_timstamp_delta_ms; + uint16_t network2_timstamp_delta_ms; + bool is_timing_frame; +}; + +} // namespace webrtc + +#endif // WEBRTC_API_VIDEO_VIDEO_TIMING_H_ diff --git a/webrtc/common_types.cc b/webrtc/common_types.cc index 5cbc452db3..0511f0db32 100644 --- a/webrtc/common_types.cc +++ b/webrtc/common_types.cc @@ -54,7 +54,8 @@ RTPHeaderExtension::RTPHeaderExtension() hasVideoRotation(false), videoRotation(kVideoRotation_0), hasVideoContentType(false), - videoContentType(VideoContentType::UNSPECIFIED) {} + videoContentType(VideoContentType::UNSPECIFIED), + has_video_timing(false) {} RTPHeader::RTPHeader() : markerBit(false), @@ -86,6 +87,7 @@ VideoCodec::VideoCodec() spatialLayers(), mode(kRealtimeVideo), expect_encode_from_texture(false), + timing_frame_thresholds({0, 0}), codec_specific_() {} VideoCodecVP8* VideoCodec::VP8() { diff --git a/webrtc/common_types.h b/webrtc/common_types.h index 73a1c8340d..b0a21d6046 100644 --- a/webrtc/common_types.h +++ b/webrtc/common_types.h @@ -20,6 +20,7 @@ #include "webrtc/api/video/video_content_type.h" #include "webrtc/api/video/video_rotation.h" +#include "webrtc/api/video/video_timing.h" #include "webrtc/base/array_view.h" #include "webrtc/base/checks.h" #include "webrtc/base/optional.h" @@ -588,6 +589,19 @@ class VideoCodec { VideoCodecMode mode; bool expect_encode_from_texture; + // Timing frames configuration. There is delay of delay_ms between two + // consequent timing frames, excluding outliers. Frame is always made a + // timing frame if it's at least outlier_ratio in percent of "ideal" average + // frame given bitrate and framerate, i.e. if it's bigger than + // |outlier_ratio / 100.0 * bitrate_bps / fps| in bits. This way, timing + // frames will not be sent too often usually. Yet large frames will always + // have timing information for debug purposes because they are more likely to + // cause extra delays. + struct TimingFrameTriggerThresholds { + int64_t delay_ms; + uint16_t outlier_ratio_percent; + } timing_frame_thresholds; + bool operator==(const VideoCodec& other) const = delete; bool operator!=(const VideoCodec& other) const = delete; @@ -763,6 +777,9 @@ struct RTPHeaderExtension { bool hasVideoContentType; VideoContentType videoContentType; + bool has_video_timing; + VideoTiming video_timing; + PlayoutDelay playout_delay = {-1, -1}; // For identification of a stream when ssrc is not signaled. See diff --git a/webrtc/common_video/include/video_frame.h b/webrtc/common_video/include/video_frame.h index 1e8f37c171..10dfb47cc9 100644 --- a/webrtc/common_video/include/video_frame.h +++ b/webrtc/common_video/include/video_frame.h @@ -35,6 +35,12 @@ class EncodedImage { EncodedImage(uint8_t* buffer, size_t length, size_t size) : _buffer(buffer), _length(length), _size(size) {} + void SetEncodeTime(int64_t encode_start_ms, int64_t encode_finish_ms) const { + timing_.is_timing_frame = true; + timing_.encode_start_ms = encode_start_ms; + timing_.encode_finish_ms = encode_finish_ms; + } + // TODO(kthelgason): get rid of this struct as it only has a single member // remaining. struct AdaptReason { @@ -63,6 +69,19 @@ class EncodedImage { // indication that all future frames will be constrained with those limits // until the application indicates a change again. PlayoutDelay playout_delay_ = {-1, -1}; + + // Timing information should be updatable on const instances. + mutable struct Timing { + bool is_timing_frame = false; + int64_t encode_start_ms = 0; + int64_t encode_finish_ms = 0; + int64_t packetization_finish_ms = 0; + int64_t pacer_exit_ms = 0; + int64_t network_timestamp_ms = 0; + int64_t network2_timestamp_ms = 0; + int64_t receive_start_ms = 0; + int64_t receive_finish_ms = 0; + } timing_; }; } // namespace webrtc diff --git a/webrtc/config.cc b/webrtc/config.cc index 616ca6e76a..1a2c13d044 100644 --- a/webrtc/config.cc +++ b/webrtc/config.cc @@ -76,6 +76,10 @@ const char* RtpExtension::kVideoContentTypeUri = "http://www.webrtc.org/experiments/rtp-hdrext/video-content-type"; const int RtpExtension::kVideoContentTypeDefaultId = 7; +const char* RtpExtension::kVideoTimingUri = + "http://www.webrtc.org/experiments/rtp-hdrext/video-timing"; +const int RtpExtension::kVideoTimingDefaultId = 8; + const int RtpExtension::kMinId = 1; const int RtpExtension::kMaxId = 14; @@ -90,7 +94,8 @@ bool RtpExtension::IsSupportedForVideo(const std::string& uri) { uri == webrtc::RtpExtension::kVideoRotationUri || uri == webrtc::RtpExtension::kTransportSequenceNumberUri || uri == webrtc::RtpExtension::kPlayoutDelayUri || - uri == webrtc::RtpExtension::kVideoContentTypeUri; + uri == webrtc::RtpExtension::kVideoContentTypeUri || + uri == webrtc::RtpExtension::kVideoTimingUri; } VideoStream::VideoStream() diff --git a/webrtc/config.h b/webrtc/config.h index f0039b3a72..d5ed266c89 100644 --- a/webrtc/config.h +++ b/webrtc/config.h @@ -92,6 +92,10 @@ struct RtpExtension { static const char* kVideoContentTypeUri; static const int kVideoContentTypeDefaultId; + // Header extension for video timing. + static const char* kVideoTimingUri; + static const int kVideoTimingDefaultId; + // Header extension for transport sequence number, see url for details: // http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions static const char* kTransportSequenceNumberUri; diff --git a/webrtc/media/engine/webrtcvideoengine.cc b/webrtc/media/engine/webrtcvideoengine.cc index 54d7103e16..e592740bb9 100644 --- a/webrtc/media/engine/webrtcvideoengine.cc +++ b/webrtc/media/engine/webrtcvideoengine.cc @@ -458,6 +458,8 @@ RtpCapabilities WebRtcVideoEngine::GetCapabilities() const { webrtc::RtpExtension(webrtc::RtpExtension::kVideoContentTypeUri, webrtc::RtpExtension::kVideoContentTypeDefaultId)); } + // TODO(ilnik): Add kVideoTimingUri/kVideoTimingDefaultId to capabilities. + // Possibly inside field trial. return capabilities; } diff --git a/webrtc/modules/include/module_common_types.h b/webrtc/modules/include/module_common_types.h index f4d42a8518..b719226d24 100644 --- a/webrtc/modules/include/module_common_types.h +++ b/webrtc/modules/include/module_common_types.h @@ -61,6 +61,8 @@ struct RTPVideoHeader { VideoContentType content_type; + VideoTiming video_timing; + bool is_first_packet_in_frame; uint8_t simulcastIdx; // Index if the simulcast encoder creating // this frame, 0 if not using simulcast. diff --git a/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 90537c3e30..56f03d5409 100644 --- a/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -77,6 +77,7 @@ enum RTPExtensionType { kRtpExtensionTransportSequenceNumber, kRtpExtensionPlayoutDelay, kRtpExtensionVideoContentType, + kRtpExtensionVideoTiming, kRtpExtensionRtpStreamId, kRtpExtensionRepairedRtpStreamId, kRtpExtensionNumberOfExtensions // Must be the last entity in the enum. diff --git a/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc index 4df13b8eea..9b30a5e596 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -39,6 +39,7 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), + CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), }; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc b/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc index 2f3feb34cd..f630bafe28 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -245,6 +245,72 @@ bool VideoContentTypeExtension::Write(uint8_t* data, return true; } +// Video Timing. +// 6 timestamps in milliseconds counted from capture time stored in rtp header: +// encode start/finish, packetization complete, pacer exit and reserved for +// modification by the network modification. +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | len=11| encode start ms delta | encode finish | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ms delta | packetizer finish ms delta | pacer exit | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ms delta | network timestamp ms delta | network2 time-| +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | stamp ms delta| +// +-+-+-+-+-+-+-+-+ + +constexpr RTPExtensionType VideoTimingExtension::kId; +constexpr uint8_t VideoTimingExtension::kValueSizeBytes; +constexpr const char* VideoTimingExtension::kUri; + +bool VideoTimingExtension::Parse(rtc::ArrayView data, + VideoTiming* timing) { + RTC_DCHECK(timing); + if (data.size() != kValueSizeBytes) + return false; + timing->encode_start_delta_ms = + ByteReader::ReadBigEndian(data.data()); + timing->encode_finish_delta_ms = ByteReader::ReadBigEndian( + data.data() + 2 * VideoTiming::kEncodeFinishDeltaIdx); + timing->packetization_finish_delta_ms = ByteReader::ReadBigEndian( + data.data() + 2 * VideoTiming::kPacketizationFinishDeltaIdx); + timing->pacer_exit_delta_ms = ByteReader::ReadBigEndian( + data.data() + 2 * VideoTiming::kPacerExitDeltaIdx); + timing->network_timstamp_delta_ms = ByteReader::ReadBigEndian( + data.data() + 2 * VideoTiming::kNetworkTimestampDeltaIdx); + timing->network2_timstamp_delta_ms = ByteReader::ReadBigEndian( + data.data() + 2 * VideoTiming::kNetwork2TimestampDeltaIdx); + timing->is_timing_frame = true; + return true; +} + +bool VideoTimingExtension::Write(uint8_t* data, const VideoTiming& timing) { + ByteWriter::WriteBigEndian(data, timing.encode_start_delta_ms); + ByteWriter::WriteBigEndian( + data + 2 * VideoTiming::kEncodeFinishDeltaIdx, + timing.encode_finish_delta_ms); + ByteWriter::WriteBigEndian( + data + 2 * VideoTiming::kPacketizationFinishDeltaIdx, + timing.packetization_finish_delta_ms); + ByteWriter::WriteBigEndian( + data + 2 * VideoTiming::kPacerExitDeltaIdx, timing.pacer_exit_delta_ms); + ByteWriter::WriteBigEndian( + data + 2 * VideoTiming::kNetworkTimestampDeltaIdx, 0); // reserved + ByteWriter::WriteBigEndian( + data + 2 * VideoTiming::kNetwork2TimestampDeltaIdx, 0); // reserved + return true; +} + +bool VideoTimingExtension::Write(uint8_t* data, + uint16_t time_delta_ms, + uint8_t idx) { + RTC_DCHECK_LT(idx, 6); + ByteWriter::WriteBigEndian(data + 2 * idx, time_delta_ms); + return true; +} + // RtpStreamId. constexpr RTPExtensionType RtpStreamId::kId; constexpr const char* RtpStreamId::kUri; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h b/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h index d3e72ebd22..c0a4bca913 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -15,6 +15,7 @@ #include "webrtc/api/video/video_content_type.h" #include "webrtc/api/video/video_rotation.h" +#include "webrtc/api/video/video_timing.h" #include "webrtc/base/array_view.h" #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h" @@ -126,6 +127,24 @@ class VideoContentTypeExtension { static bool Write(uint8_t* data, VideoContentType content_type); }; +class VideoTimingExtension { + public: + static constexpr RTPExtensionType kId = kRtpExtensionVideoTiming; + static constexpr uint8_t kValueSizeBytes = 12; + static constexpr const char* kUri = + "http://www.webrtc.org/experiments/rtp-hdrext/video-timing"; + + static bool Parse(rtc::ArrayView data, VideoTiming* timing); + static size_t ValueSize(const VideoTiming&) { return kValueSizeBytes; } + static bool Write(uint8_t* data, const VideoTiming& timing); + + static size_t ValueSize(uint16_t time_delta_ms, uint8_t idx) { + return kValueSizeBytes; + } + // Writes only single time delta to position idx. + static bool Write(uint8_t* data, uint16_t time_delta_ms, uint8_t idx); +}; + class RtpStreamId { public: static constexpr RTPExtensionType kId = kRtpExtensionRtpStreamId; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_packet.cc b/webrtc/modules/rtp_rtcp/source/rtp_packet.cc index 8c6a9e2593..0ccf05e3bf 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_packet.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_packet.cc @@ -174,6 +174,8 @@ void Packet::GetHeader(RTPHeader* header) const { header->extension.hasVideoContentType = GetExtension( &header->extension.videoContentType); + header->extension.has_video_timing = + GetExtension(&header->extension.video_timing); GetExtension(&header->extension.stream_id); GetExtension(&header->extension.repaired_stream_id); GetExtension(&header->extension.playout_delay); diff --git a/webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h b/webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h index f2ddc8a52e..55bad2d502 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h +++ b/webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h @@ -10,6 +10,7 @@ #ifndef WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_PACKET_TO_SEND_H_ #define WEBRTC_MODULES_RTP_RTCP_SOURCE_RTP_PACKET_TO_SEND_H_ +#include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h" #include "webrtc/modules/rtp_rtcp/source/rtp_packet.h" namespace webrtc { @@ -23,10 +24,36 @@ class RtpPacketToSend : public rtp::Packet { : Packet(extensions, capacity) {} RtpPacketToSend& operator=(const RtpPacketToSend& packet) = default; + // Time in local time base as close as it can to frame capture time. int64_t capture_time_ms() const { return capture_time_ms_; } + void set_capture_time_ms(int64_t time) { capture_time_ms_ = time; } + void set_packetization_finish_time_ms(int64_t time) { + SetExtension( + VideoTiming::GetDeltaCappedMs(capture_time_ms_, time), + VideoTiming::kPacketizationFinishDeltaIdx); + } + + void set_pacer_exit_time_ms(int64_t time) { + SetExtension( + VideoTiming::GetDeltaCappedMs(capture_time_ms_, time), + VideoTiming::kPacerExitDeltaIdx); + } + + void set_network_time_ms(int64_t time) { + SetExtension( + VideoTiming::GetDeltaCappedMs(capture_time_ms_, time), + VideoTiming::kNetworkTimestampDeltaIdx); + } + + void set_network2_time_ms(int64_t time) { + SetExtension( + VideoTiming::GetDeltaCappedMs(capture_time_ms_, time), + VideoTiming::kNetwork2TimestampDeltaIdx); + } + private: int64_t capture_time_ms_ = 0; }; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_receiver_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_receiver_video.cc index debe836d8d..0edc47824c 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_receiver_video.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_receiver_video.cc @@ -91,6 +91,7 @@ int32_t RTPReceiverVideo::ParseRtpPacket(WebRtcRTPHeader* rtp_header, rtp_header->type = parsed_payload.type; rtp_header->type.Video.rotation = kVideoRotation_0; rtp_header->type.Video.content_type = VideoContentType::UNSPECIFIED; + rtp_header->type.Video.video_timing.is_timing_frame = false; // Retrieve the video rotation information. if (rtp_header->header.extension.hasVideoRotation) { @@ -103,6 +104,12 @@ int32_t RTPReceiverVideo::ParseRtpPacket(WebRtcRTPHeader* rtp_header, rtp_header->header.extension.videoContentType; } + if (rtp_header->header.extension.has_video_timing) { + rtp_header->type.Video.video_timing = + rtp_header->header.extension.video_timing; + rtp_header->type.Video.video_timing.is_timing_frame = true; + } + rtp_header->type.Video.playout_delay = rtp_header->header.extension.playout_delay; diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index faaf4c4f78..a6c7647be5 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -42,6 +42,8 @@ RTPExtensionType StringToRtpExtensionType(const std::string& extension) { return kRtpExtensionPlayoutDelay; if (extension == RtpExtension::kVideoContentTypeUri) return kRtpExtensionVideoContentType; + if (extension == RtpExtension::kVideoTimingUri) + return kRtpExtensionVideoTiming; RTC_NOTREACHED() << "Looking up unsupported RTP extension."; return kRtpExtensionNone; } diff --git a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index 783fe5eadd..73bdd7a5ab 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -214,6 +214,7 @@ class RtpRtcpImplTest : public ::testing::Test { rtp_video_header.simulcastIdx = 0; rtp_video_header.codec = kRtpVideoVp8; rtp_video_header.codecHeader = {vp8_header}; + rtp_video_header.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; const uint8_t payload[100] = {0}; EXPECT_EQ(true, module->impl_->SendOutgoingData( diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index 00edd18f3f..cdd7079781 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -743,6 +743,9 @@ bool RTPSender::PrepareAndSendPacket(std::unique_ptr packet, packet_to_send->SetExtension( AbsoluteSendTime::MsTo24Bits(now_ms)); + if (packet_to_send->HasExtension()) + packet_to_send->set_pacer_exit_time_ms(now_ms); + PacketOptions options; if (UpdateTransportSequenceNumber(packet_to_send, &options.packet_id)) { AddPacketToTransportFeedback(options.packet_id, *packet_to_send, @@ -830,6 +833,8 @@ bool RTPSender::SendToNetwork(std::unique_ptr packet, if (packet->capture_time_ms() > 0) { packet->SetExtension( kTimestampTicksPerMs * (now_ms - packet->capture_time_ms())); + if (packet->HasExtension()) + packet->set_pacer_exit_time_ms(now_ms); } packet->SetExtension(AbsoluteSendTime::MsTo24Bits(now_ms)); diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc index d44cf683dc..3c5c116cb0 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -38,6 +38,7 @@ namespace { const int kTransmissionTimeOffsetExtensionId = 1; const int kAbsoluteSendTimeExtensionId = 14; const int kTransportSequenceNumberExtensionId = 13; +const int kVideoTimingExtensionId = 12; const int kPayload = 100; const int kRtxPayload = 98; const uint32_t kTimestamp = 10; @@ -74,6 +75,8 @@ class LoopbackTransportTest : public webrtc::Transport { kVideoRotationExtensionId); receivers_extensions_.Register(kRtpExtensionAudioLevel, kAudioLevelExtensionId); + receivers_extensions_.Register(kRtpExtensionVideoTiming, + kVideoTimingExtensionId); } bool SendRtp(const uint8_t* data, @@ -460,6 +463,51 @@ TEST_P(RtpSenderTest, SendsPacketsWithTransportSequenceNumber) { EXPECT_EQ(transport_.last_packet_id_, transport_seq_no); } +TEST_P(RtpSenderTestWithoutPacer, WritesTimestampToTimingExtension) { + rtp_sender_->SetStorePacketsStatus(true, 10); + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoTiming, kVideoTimingExtensionId)); + int64_t capture_time_ms = fake_clock_.TimeInMilliseconds(); + auto packet = rtp_sender_->AllocatePacket(); + packet->SetPayloadType(kPayload); + packet->SetMarker(true); + packet->SetTimestamp(kTimestamp); + packet->set_capture_time_ms(capture_time_ms); + const VideoTiming kVideoTiming = {0u, 0u, 0u, 0u, 0u, 0u, true}; + packet->SetExtension(kVideoTiming); + EXPECT_TRUE(rtp_sender_->AssignSequenceNumber(packet.get())); + size_t packet_size = packet->size(); + webrtc::RTPHeader rtp_header; + + packet->GetHeader(&rtp_header); + + const int kStoredTimeInMs = 100; + fake_clock_.AdvanceTimeMilliseconds(kStoredTimeInMs); + + EXPECT_TRUE(rtp_sender_->SendToNetwork(std::move(packet), + kAllowRetransmission, + RtpPacketSender::kNormalPriority)); + EXPECT_EQ(1, transport_.packets_sent()); + EXPECT_EQ(packet_size, transport_.last_sent_packet().size()); + + transport_.last_sent_packet().GetHeader(&rtp_header); + EXPECT_TRUE(rtp_header.extension.has_video_timing); + EXPECT_EQ(kStoredTimeInMs, + rtp_header.extension.video_timing.pacer_exit_delta_ms); + + fake_clock_.AdvanceTimeMilliseconds(kStoredTimeInMs); + rtp_sender_->TimeToSendPacket(kSsrc, kSeqNum, capture_time_ms, false, + PacedPacketInfo()); + + EXPECT_EQ(2, transport_.packets_sent()); + EXPECT_EQ(packet_size, transport_.last_sent_packet().size()); + + transport_.last_sent_packet().GetHeader(&rtp_header); + EXPECT_TRUE(rtp_header.extension.has_video_timing); + EXPECT_EQ(kStoredTimeInMs * 2, + rtp_header.extension.video_timing.pacer_exit_delta_ms); +} + TEST_P(RtpSenderTest, TrafficSmoothingWithExtensions) { EXPECT_CALL(mock_paced_sender_, InsertPacket(RtpPacketSender::kNormalPriority, kSsrc, kSeqNum, _, _, _)); @@ -1410,6 +1458,33 @@ TEST_P(RtpSenderVideoTest, KeyFrameHasCVO) { EXPECT_EQ(kVideoRotation_0, rotation); } +TEST_P(RtpSenderVideoTest, TimingFrameHasPacketizationTimstampSet) { + uint8_t kFrame[kMaxPacketLength]; + const int64_t kPacketizationTimeMs = 100; + const int64_t kEncodeStartDeltaMs = 10; + const int64_t kEncodeFinishDeltaMs = 50; + EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( + kRtpExtensionVideoTiming, kVideoTimingExtensionId)); + + const int64_t kCaptureTimestamp = fake_clock_.TimeInMilliseconds(); + + RTPVideoHeader hdr = {0}; + hdr.video_timing.is_timing_frame = true; + hdr.video_timing.encode_start_delta_ms = kEncodeStartDeltaMs; + hdr.video_timing.encode_finish_delta_ms = kEncodeFinishDeltaMs; + + fake_clock_.AdvanceTimeMilliseconds(kPacketizationTimeMs); + rtp_sender_video_->SendVideo(kRtpVideoGeneric, kVideoFrameKey, kPayload, + kTimestamp, kCaptureTimestamp, kFrame, + sizeof(kFrame), nullptr, &hdr); + VideoTiming timing; + EXPECT_TRUE(transport_.last_sent_packet().GetExtension( + &timing)); + EXPECT_EQ(kPacketizationTimeMs, timing.packetization_finish_delta_ms); + EXPECT_EQ(kEncodeStartDeltaMs, timing.encode_start_delta_ms); + EXPECT_EQ(kEncodeFinishDeltaMs, timing.encode_finish_delta_ms); +} + TEST_P(RtpSenderVideoTest, DeltaFrameHasCVOWhenChanged) { uint8_t kFrame[kMaxPacketLength]; EXPECT_EQ(0, rtp_sender_->RegisterRtpHeaderExtension( diff --git a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc index 3f4c401a19..41af62b9fa 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -304,6 +304,7 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, auto last_packet = rtc::MakeUnique(*rtp_header); size_t fec_packet_overhead; + bool is_timing_frame = false; bool red_enabled; int32_t retransmission_settings; { @@ -332,6 +333,11 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, last_packet->SetExtension( video_header->content_type); } + if (video_header->video_timing.is_timing_frame) { + last_packet->SetExtension( + video_header->video_timing); + is_timing_frame = true; + } } // FEC settings. @@ -388,6 +394,11 @@ bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type, if (!rtp_sender_->AssignSequenceNumber(packet.get())) return false; + // Put packetization finish timestamp into extension. + if (last && is_timing_frame) { + packet->set_packetization_finish_time_ms(clock_->TimeInMilliseconds()); + } + const bool protect_packet = (packetizer->GetProtectionType() == kProtectedPacket); if (flexfec_enabled()) { diff --git a/webrtc/modules/rtp_rtcp/source/rtp_utility.cc b/webrtc/modules/rtp_rtcp/source/rtp_utility.cc index 06786cb27a..f959087c99 100644 --- a/webrtc/modules/rtp_rtcp/source/rtp_utility.cc +++ b/webrtc/modules/rtp_rtcp/source/rtp_utility.cc @@ -10,8 +10,6 @@ #include "webrtc/modules/rtp_rtcp/source/rtp_utility.h" -#include - #include "webrtc/base/logging.h" #include "webrtc/modules/rtp_rtcp/include/rtp_cvo.h" #include "webrtc/modules/rtp_rtcp/source/byte_io.h" @@ -253,6 +251,9 @@ bool RtpHeaderParser::Parse(RTPHeader* header, header->extension.hasVideoContentType = false; header->extension.videoContentType = VideoContentType::UNSPECIFIED; + header->extension.has_video_timing = false; + header->extension.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; + if (X) { /* RTP header extension, RFC 3550. 0 1 2 3 @@ -464,6 +465,16 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( } break; } + case kRtpExtensionVideoTiming: { + if (len != VideoTimingExtension::kValueSizeBytes - 1) { + LOG(LS_WARNING) << "Incorrect video timing len: " << len; + return; + } + header->extension.has_video_timing = true; + VideoTimingExtension::Parse(rtc::MakeArrayView(ptr, len + 1), + &header->extension.video_timing); + break; + } case kRtpExtensionRtpStreamId: { header->extension.stream_id.Set(rtc::MakeArrayView(ptr, len + 1)); break; diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index 2148f8ac62..d333264d93 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -513,6 +513,7 @@ if (rtc_include_tests) { "codecs/vp8/simulcast_unittest.h", "decoding_state_unittest.cc", "frame_buffer2_unittest.cc", + "generic_encoder_unittest.cc", "h264_sprop_parameter_sets_unittest.cc", "h264_sps_pps_tracker_unittest.cc", "histogram_unittest.cc", diff --git a/webrtc/modules/video_coding/codec_database.cc b/webrtc/modules/video_coding/codec_database.cc index 794e955723..92ae5d23b5 100644 --- a/webrtc/modules/video_coding/codec_database.cc +++ b/webrtc/modules/video_coding/codec_database.cc @@ -127,6 +127,9 @@ void VCMCodecDataBase::Codec(VideoCodecType codec_type, VideoCodec* settings) { settings->height = VCM_DEFAULT_CODEC_HEIGHT; settings->numberOfSimulcastStreams = 0; settings->qpMax = 56; + settings->timing_frame_thresholds = { + kDefaultTimingFramesDelayMs, kDefaultOutlierFrameSizePercent, + }; *(settings->VP8()) = VideoEncoder::GetDefaultVp8Settings(); return; case kVideoCodecVP9: @@ -142,6 +145,9 @@ void VCMCodecDataBase::Codec(VideoCodecType codec_type, VideoCodec* settings) { settings->height = VCM_DEFAULT_CODEC_HEIGHT; settings->numberOfSimulcastStreams = 0; settings->qpMax = 56; + settings->timing_frame_thresholds = { + kDefaultTimingFramesDelayMs, kDefaultOutlierFrameSizePercent, + }; *(settings->VP9()) = VideoEncoder::GetDefaultVp9Settings(); return; case kVideoCodecH264: @@ -157,6 +163,9 @@ void VCMCodecDataBase::Codec(VideoCodecType codec_type, VideoCodec* settings) { settings->height = VCM_DEFAULT_CODEC_HEIGHT; settings->numberOfSimulcastStreams = 0; settings->qpMax = 56; + settings->timing_frame_thresholds = { + kDefaultTimingFramesDelayMs, kDefaultOutlierFrameSizePercent, + }; *(settings->H264()) = VideoEncoder::GetDefaultH264Settings(); return; case kVideoCodecI420: diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc index dbc83f8eee..cdbe6f5ecf 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -21,8 +21,8 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/base/timeutils.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/media/base/mediaconstants.h" #include "webrtc/system_wrappers/include/metrics.h" namespace webrtc { @@ -370,6 +370,7 @@ int32_t H264EncoderImpl::Encode(const VideoFrame& input_frame, encoded_image_.content_type_ = (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; + encoded_image_.timing_.is_timing_frame = false; encoded_image_._frameType = ConvertToVideoFrameType(info.eFrameType); // Split encoded image up into fragments. This also updates |encoded_image_|. diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h index d097dcf229..5a8d21913e 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h @@ -25,6 +25,7 @@ #include "webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h" +#include "webrtc/modules/video_coding/include/video_coding_defines.h" #include "webrtc/test/gtest.h" using ::testing::_; @@ -206,6 +207,8 @@ class TestVp8Simulcast : public ::testing::Test { settings->height = kDefaultHeight; settings->numberOfSimulcastStreams = kNumberOfSimulcastStreams; ASSERT_EQ(3, kNumberOfSimulcastStreams); + settings->timing_frame_thresholds = {kDefaultTimingFramesDelayMs, + kDefaultOutlierFrameSizePercent}; ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, kMaxBitrates[0], kMinBitrates[0], kTargetBitrates[0], &settings->simulcastStream[0], temporal_layer_profile[0]); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 74c02a535c..5038e8edf4 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -878,6 +878,7 @@ int VP8EncoderImpl::GetEncodedPartitions( encoded_images_[encoder_idx].content_type_ = (codec_.mode == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; + encoded_images_[encoder_idx].timing_.is_timing_frame = false; int qp = -1; vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index fa149ef70f..8e4560a720 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -29,7 +29,6 @@ #include "webrtc/base/trace_event.h" #include "webrtc/common_video/include/video_frame_buffer.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" -#include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/video_coding/codecs/vp9/screenshare_layers.h" namespace webrtc { @@ -710,9 +709,11 @@ int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) { : VideoContentType::UNSPECIFIED; encoded_image_._encodedHeight = raw_->d_h; encoded_image_._encodedWidth = raw_->d_w; + encoded_image_.timing_.is_timing_frame = false; int qp = -1; vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp); encoded_image_.qp_ = qp; + encoded_complete_callback_->OnEncodedImage(encoded_image_, &codec_specific, &frag_info); } diff --git a/webrtc/modules/video_coding/encoded_frame.cc b/webrtc/modules/video_coding/encoded_frame.cc index 1807fa5c4c..ec7f560755 100644 --- a/webrtc/modules/video_coding/encoded_frame.cc +++ b/webrtc/modules/video_coding/encoded_frame.cc @@ -88,6 +88,7 @@ void VCMEncodedFrame::Reset() { _codec = kVideoCodecUnknown; rotation_ = kVideoRotation_0; content_type_ = VideoContentType::UNSPECIFIED; + timing_.is_timing_frame = false; _rotation_set = false; } diff --git a/webrtc/modules/video_coding/encoded_frame.h b/webrtc/modules/video_coding/encoded_frame.h index 2d65e9c1e4..7956485f21 100644 --- a/webrtc/modules/video_coding/encoded_frame.h +++ b/webrtc/modules/video_coding/encoded_frame.h @@ -85,6 +85,10 @@ class VCMEncodedFrame : protected EncodedImage { * Get video content type */ VideoContentType contentType() const { return content_type_; } + /** + * Get video timing + */ + EncodedImage::Timing video_timing() const { return timing_; } /** * True if this frame is complete, false otherwise */ diff --git a/webrtc/modules/video_coding/frame_buffer.cc b/webrtc/modules/video_coding/frame_buffer.cc index 5ea12dc0e6..11bf88a766 100644 --- a/webrtc/modules/video_coding/frame_buffer.cc +++ b/webrtc/modules/video_coding/frame_buffer.cc @@ -164,6 +164,27 @@ VCMFrameBufferEnum VCMFrameBuffer::InsertPacket( rotation_ = packet.video_header.rotation; _rotation_set = true; content_type_ = packet.video_header.content_type; + if (packet.video_header.video_timing.is_timing_frame) { + timing_.is_timing_frame = true; + timing_.encode_start_ms = + ntp_time_ms_ + packet.video_header.video_timing.encode_start_delta_ms; + timing_.encode_finish_ms = + ntp_time_ms_ + + packet.video_header.video_timing.encode_finish_delta_ms; + timing_.packetization_finish_ms = + ntp_time_ms_ + + packet.video_header.video_timing.packetization_finish_delta_ms; + timing_.pacer_exit_ms = + ntp_time_ms_ + packet.video_header.video_timing.pacer_exit_delta_ms; + timing_.network_timestamp_ms = + ntp_time_ms_ + + packet.video_header.video_timing.network_timstamp_delta_ms; + timing_.network2_timestamp_ms = + ntp_time_ms_ + + packet.video_header.video_timing.network2_timstamp_delta_ms; + } else { + timing_.is_timing_frame = false; + } } if (packet.is_first_packet_in_frame) { diff --git a/webrtc/modules/video_coding/frame_object.cc b/webrtc/modules/video_coding/frame_object.cc index f2a5ab2719..220fa534d1 100644 --- a/webrtc/modules/video_coding/frame_object.cc +++ b/webrtc/modules/video_coding/frame_object.cc @@ -111,6 +111,34 @@ RtpFrameObject::RtpFrameObject(PacketBuffer* packet_buffer, rotation_ = last_packet->video_header.rotation; _rotation_set = true; content_type_ = last_packet->video_header.content_type; + if (last_packet->video_header.video_timing.is_timing_frame) { + // ntp_time_ms_ may be -1 if not estimated yet. This is not a problem, + // as this will be dealt with at the time of reporting. + timing_.is_timing_frame = true; + timing_.encode_start_ms = + ntp_time_ms_ + + last_packet->video_header.video_timing.encode_start_delta_ms; + timing_.encode_finish_ms = + ntp_time_ms_ + + last_packet->video_header.video_timing.encode_finish_delta_ms; + timing_.packetization_finish_ms = + ntp_time_ms_ + + last_packet->video_header.video_timing.packetization_finish_delta_ms; + timing_.pacer_exit_ms = + ntp_time_ms_ + + last_packet->video_header.video_timing.pacer_exit_delta_ms; + timing_.network_timestamp_ms = + ntp_time_ms_ + + last_packet->video_header.video_timing.network_timstamp_delta_ms; + timing_.network2_timestamp_ms = + ntp_time_ms_ + + last_packet->video_header.video_timing.network2_timstamp_delta_ms; + + timing_.receive_start_ms = first_packet->receive_time_ms; + timing_.receive_finish_ms = last_packet->receive_time_ms; + } else { + timing_.is_timing_frame = false; + } } RtpFrameObject::~RtpFrameObject() { diff --git a/webrtc/modules/video_coding/generic_decoder.cc b/webrtc/modules/video_coding/generic_decoder.cc index 5969f23696..722b0b4154 100644 --- a/webrtc/modules/video_coding/generic_decoder.cc +++ b/webrtc/modules/video_coding/generic_decoder.cc @@ -24,7 +24,10 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, : _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength), - _lastReceivedPictureID(0) {} + _lastReceivedPictureID(0) { + ntp_offset_ = + _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); +} VCMDecodedFrameCallback::~VCMDecodedFrameCallback() { } @@ -85,6 +88,30 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, _timing->StopDecodeTimer(decodedImage.timestamp(), *decode_time_ms, now_ms, frameInfo->renderTimeMs); + // Report timing information. + if (frameInfo->timing.is_timing_frame) { + // Convert remote timestamps to local time from ntp timestamps. + frameInfo->timing.encode_start_ms -= ntp_offset_; + frameInfo->timing.encode_finish_ms -= ntp_offset_; + frameInfo->timing.packetization_finish_ms -= ntp_offset_; + frameInfo->timing.pacer_exit_ms -= ntp_offset_; + frameInfo->timing.network_timestamp_ms -= ntp_offset_; + frameInfo->timing.network2_timestamp_ms -= ntp_offset_; + // TODO(ilnik): Report timing information here. + // Capture time: decodedImage.ntp_time_ms() - ntp_offset + // Encode start: frameInfo->timing.encode_start_ms + // Encode finish: frameInfo->timing.encode_finish_ms + // Packetization done: frameInfo->timing.packetization_finish_ms + // Pacer exit: frameInfo->timing.pacer_exit_ms + // Network timestamp: frameInfo->timing.network_timestamp_ms + // Network2 timestamp: frameInfo->timing.network2_timestamp_ms + // Receive start: frameInfo->timing.receive_start_ms + // Receive finish: frameInfo->timing.receive_finish_ms + // Decode start: frameInfo->decodeStartTimeMs + // Decode finish: now_ms + // Render time: frameInfo->renderTimeMs + } + decodedImage.set_timestamp_us( frameInfo->renderTimeMs * rtc::kNumMicrosecsPerMillisec); decodedImage.set_rotation(frameInfo->rotation); @@ -151,6 +178,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); + _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); // Set correctly only for key frames. Thus, use latest key frame // content type. If the corresponding key frame was lost, decode will fail // and content type will be ignored. diff --git a/webrtc/modules/video_coding/generic_decoder.h b/webrtc/modules/video_coding/generic_decoder.h index 71b8d81c74..8e26a8b1af 100644 --- a/webrtc/modules/video_coding/generic_decoder.h +++ b/webrtc/modules/video_coding/generic_decoder.h @@ -31,6 +31,7 @@ struct VCMFrameInformation { void* userData; VideoRotation rotation; VideoContentType content_type; + EncodedImage::Timing timing; }; class VCMDecodedFrameCallback : public DecodedImageCallback { @@ -68,6 +69,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { rtc::CriticalSection lock_; VCMTimestampMap _timestampMap GUARDED_BY(lock_); uint64_t _lastReceivedPictureID; + int64_t ntp_offset_; }; class VCMGenericDecoder { diff --git a/webrtc/modules/video_coding/generic_encoder.cc b/webrtc/modules/video_coding/generic_encoder.cc index 50731c050a..5faa193f9f 100644 --- a/webrtc/modules/video_coding/generic_encoder.cc +++ b/webrtc/modules/video_coding/generic_encoder.cc @@ -15,6 +15,7 @@ #include "webrtc/api/video/i420_buffer.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" +#include "webrtc/base/timeutils.h" #include "webrtc/base/trace_event.h" #include "webrtc/modules/video_coding/encoded_frame.h" #include "webrtc/modules/video_coding/media_optimization.h" @@ -29,7 +30,8 @@ VCMGenericEncoder::VCMGenericEncoder( vcm_encoded_frame_callback_(encoded_frame_callback), internal_source_(internal_source), encoder_params_({BitrateAllocation(), 0, 0, 0}), - is_screenshare_(false) {} + is_screenshare_(false), + streams_or_svc_num_(0) {} VCMGenericEncoder::~VCMGenericEncoder() {} @@ -45,6 +47,17 @@ int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings, RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode"); is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing; + streams_or_svc_num_ = settings->numberOfSimulcastStreams; + if (settings->codecType == kVideoCodecVP9) { + streams_or_svc_num_ = settings->VP9().numberOfSpatialLayers; + } + if (streams_or_svc_num_ == 0) + streams_or_svc_num_ = 1; + + vcm_encoded_frame_callback_->SetTimingFramesThresholds( + settings->timing_frame_thresholds); + vcm_encoded_frame_callback_->OnFrameRateChanged(settings->maxFramerate); + if (encoder_->InitEncode(settings, number_of_cores, max_payload_size) != 0) { LOG(LS_ERROR) << "Failed to initialize the encoder associated with " "payload name: " @@ -65,6 +78,8 @@ int32_t VCMGenericEncoder::Encode(const VideoFrame& frame, for (FrameType frame_type : frame_types) RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta); + for (size_t i = 0; i < streams_or_svc_num_; ++i) + vcm_encoded_frame_callback_->OnEncodeStarted(frame.render_time_ms(), i); int32_t result = encoder_->Encode(frame, codec_specific, &frame_types); if (is_screenshare_ && @@ -107,6 +122,17 @@ void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) { << ", framerate = " << params.input_frame_rate << "): " << res; } + vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate); + for (size_t i = 0; i < streams_or_svc_num_; ++i) { + size_t layer_bitrate_bytes_per_sec = + params.target_bitrate.GetSpatialLayerSum(i) / 8; + // VP9 rate control is not yet moved out of VP9Impl. Due to that rates + // are not split among spatial layers. + if (layer_bitrate_bytes_per_sec == 0) + layer_bitrate_bytes_per_sec = params.target_bitrate.get_sum_bps() / 8; + vcm_encoded_frame_callback_->OnTargetBitrateChanged( + layer_bitrate_bytes_per_sec, i); + } } } @@ -124,6 +150,8 @@ int32_t VCMGenericEncoder::RequestFrame( const std::vector& frame_types) { RTC_DCHECK_RUNS_SERIALIZED(&race_checker_); + for (size_t i = 0; i < streams_or_svc_num_; ++i) + vcm_encoded_frame_callback_->OnEncodeStarted(0, i); // TODO(nisse): Used only with internal source. Delete as soon as // that feature is removed. The only implementation I've been able // to find ignores what's in the frame. With one exception: It seems @@ -151,16 +179,102 @@ VCMEncodedFrameCallback::VCMEncodedFrameCallback( media_optimization::MediaOptimization* media_opt) : internal_source_(false), post_encode_callback_(post_encode_callback), - media_opt_(media_opt) {} + media_opt_(media_opt), + framerate_(1), + last_timing_frame_time_ms_(-1), + timing_frames_thresholds_({-1, 0}) {} VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {} +void VCMEncodedFrameCallback::OnTargetBitrateChanged( + size_t bitrate_bytes_per_second, + size_t simulcast_svc_idx) { + rtc::CritScope crit(&timing_params_lock_); + if (timing_frames_info_.size() < simulcast_svc_idx + 1) + timing_frames_info_.resize(simulcast_svc_idx + 1); + timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec = + bitrate_bytes_per_second; +} + +void VCMEncodedFrameCallback::OnFrameRateChanged(size_t framerate) { + rtc::CritScope crit(&timing_params_lock_); + framerate_ = framerate; +} + +void VCMEncodedFrameCallback::OnEncodeStarted(int64_t capture_time_ms, + size_t simulcast_svc_idx) { + rtc::CritScope crit(&timing_params_lock_); + if (timing_frames_info_.size() < simulcast_svc_idx + 1) + timing_frames_info_.resize(simulcast_svc_idx + 1); + timing_frames_info_[simulcast_svc_idx].encode_start_time_ms[capture_time_ms] = + rtc::TimeMillis(); +} + EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage( const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific, const RTPFragmentationHeader* fragmentation_header) { TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded", "timestamp", encoded_image._timeStamp); + bool is_timing_frame = false; + size_t outlier_frame_size = 0; + int64_t encode_start_ms = -1; + size_t simulcast_svc_idx = 0; + if (codec_specific->codecType == kVideoCodecVP9) { + if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1) + simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx; + } else if (codec_specific->codecType == kVideoCodecVP8) { + simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx; + } else if (codec_specific->codecType == kVideoCodecGeneric) { + simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx; + } else if (codec_specific->codecType == kVideoCodecH264) { + // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here. + } + + { + rtc::CritScope crit(&timing_params_lock_); + RTC_CHECK_LT(simulcast_svc_idx, timing_frames_info_.size()); + + auto encode_start_map = + &timing_frames_info_[simulcast_svc_idx].encode_start_time_ms; + auto it = encode_start_map->find(encoded_image.capture_time_ms_); + if (it != encode_start_map->end()) { + encode_start_ms = it->second; + // Assuming all encoders do not reorder frames within single stream, + // there may be some dropped frames with smaller timestamps. These should + // be purged. + encode_start_map->erase(encode_start_map->begin(), it); + encode_start_map->erase(it); + } else { + // Some chromium remoting unittests use generic encoder incorrectly + // If timestamps do not match, purge them all. + encode_start_map->erase(encode_start_map->begin(), + encode_start_map->end()); + } + + int64_t timing_frame_delay_ms = + encoded_image.capture_time_ms_ - last_timing_frame_time_ms_; + if (last_timing_frame_time_ms_ == -1 || + timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms || + timing_frame_delay_ms == 0) { + is_timing_frame = true; + last_timing_frame_time_ms_ = encoded_image.capture_time_ms_; + } + RTC_CHECK_GT(framerate_, 0); + size_t average_frame_size = + timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec / + framerate_; + outlier_frame_size = average_frame_size * + timing_frames_thresholds_.outlier_ratio_percent / 100; + } + + if (encoded_image._length >= outlier_frame_size) { + is_timing_frame = true; + } + if (encode_start_ms >= 0 && is_timing_frame) { + encoded_image.SetEncodeTime(encode_start_ms, rtc::TimeMillis()); + } + Result result = post_encode_callback_->OnEncodedImage( encoded_image, codec_specific, fragmentation_header); if (result.error != Result::OK) diff --git a/webrtc/modules/video_coding/generic_encoder.h b/webrtc/modules/video_coding/generic_encoder.h index 939d8b0fb2..8b33246eb9 100644 --- a/webrtc/modules/video_coding/generic_encoder.h +++ b/webrtc/modules/video_coding/generic_encoder.h @@ -12,6 +12,7 @@ #define WEBRTC_MODULES_VIDEO_CODING_GENERIC_ENCODER_H_ #include +#include #include #include "webrtc/modules/video_coding/include/video_codec_interface.h" @@ -44,14 +45,43 @@ class VCMEncodedFrameCallback : public EncodedImageCallback { const EncodedImage& encoded_image, const CodecSpecificInfo* codec_specific_info, const RTPFragmentationHeader* fragmentation) override; + void SetInternalSource(bool internal_source) { internal_source_ = internal_source; } + // Timing frames configuration methods. These 4 should be called before + // |OnEncodedImage| at least once. + void OnTargetBitrateChanged(size_t bitrate_bytes_per_sec, + size_t simulcast_svc_idx); + + void OnFrameRateChanged(size_t framerate); + + void OnEncodeStarted(int64_t capture_time_ms, size_t simulcast_svc_idx); + + void SetTimingFramesThresholds( + const VideoCodec::TimingFrameTriggerThresholds& thresholds) { + rtc::CritScope crit(&timing_params_lock_); + timing_frames_thresholds_ = thresholds; + } + private: + rtc::CriticalSection timing_params_lock_; bool internal_source_; EncodedImageCallback* const post_encode_callback_; media_optimization::MediaOptimization* const media_opt_; + + struct TimingFramesLayerInfo { + size_t target_bitrate_bytes_per_sec = 0; + std::map encode_start_time_ms; + }; + // Separate instance for each simulcast stream or spatial layer. + std::vector timing_frames_info_ + GUARDED_BY(timing_params_lock_); + size_t framerate_ GUARDED_BY(timing_params_lock_); + int64_t last_timing_frame_time_ms_ GUARDED_BY(timing_params_lock_); + VideoCodec::TimingFrameTriggerThresholds timing_frames_thresholds_ + GUARDED_BY(timing_params_lock_); }; class VCMGenericEncoder { @@ -88,6 +118,7 @@ class VCMGenericEncoder { rtc::CriticalSection params_lock_; EncoderParameters encoder_params_ GUARDED_BY(params_lock_); bool is_screenshare_; + size_t streams_or_svc_num_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/generic_encoder_unittest.cc b/webrtc/modules/video_coding/generic_encoder_unittest.cc new file mode 100644 index 0000000000..3de5c333fb --- /dev/null +++ b/webrtc/modules/video_coding/generic_encoder_unittest.cc @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "webrtc/modules/video_coding/encoded_frame.h" +#include "webrtc/modules/video_coding/generic_encoder.h" +#include "webrtc/modules/video_coding/include/video_coding_defines.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { +namespace test { +namespace { +inline size_t FrameSize(const size_t& min_frame_size, + const size_t& max_frame_size, + const int& s, + const int& i) { + return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size); +} + +class FakeEncodedImageCallback : public EncodedImageCallback { + public: + FakeEncodedImageCallback() : last_frame_was_timing_(false) {} + Result OnEncodedImage(const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* fragmentation) override { + last_frame_was_timing_ = encoded_image.timing_.is_timing_frame; + return Result::OK; + }; + + bool WasTimingFrame() { return last_frame_was_timing_; } + + private: + bool last_frame_was_timing_; +}; + +enum class FrameType { + kNormal, + kTiming, + kDropped, +}; + +// Emulates |num_frames| on |num_streams| frames with capture timestamps +// increased by 1 from 0. Size of each frame is between +// |min_frame_size| and |max_frame_size|, outliers are counted relatevely to +// |average_frame_sizes[]| for each stream. +std::vector> GetTimingFrames( + const int64_t delay_ms, + const size_t min_frame_size, + const size_t max_frame_size, + std::vector average_frame_sizes, + const int num_streams, + const int num_frames) { + FakeEncodedImageCallback sink; + VCMEncodedFrameCallback callback(&sink, nullptr); + const size_t kFramerate = 30; + callback.SetTimingFramesThresholds( + {delay_ms, kDefaultOutlierFrameSizePercent}); + callback.OnFrameRateChanged(kFramerate); + int s, i; + std::vector> result(num_streams); + for (s = 0; s < num_streams; ++s) + callback.OnTargetBitrateChanged(average_frame_sizes[s] * kFramerate, s); + int64_t current_timestamp = 0; + for (i = 0; i < num_frames; ++i) { + current_timestamp += 1; + for (s = 0; s < num_streams; ++s) { + // every (5+s)-th frame is dropped on s-th stream by design. + bool dropped = i % (5 + s) == 0; + + EncodedImage image; + CodecSpecificInfo codec_specific; + image._length = FrameSize(min_frame_size, max_frame_size, s, i); + image.capture_time_ms_ = current_timestamp; + codec_specific.codecType = kVideoCodecGeneric; + codec_specific.codecSpecific.generic.simulcast_idx = s; + callback.OnEncodeStarted(current_timestamp, s); + if (dropped) { + result[s].push_back(FrameType::kDropped); + continue; + } + callback.OnEncodedImage(image, &codec_specific, nullptr); + if (sink.WasTimingFrame()) { + result[s].push_back(FrameType::kTiming); + } else { + result[s].push_back(FrameType::kNormal); + } + } + } + return result; +} +} // namespace + +TEST(TestVCMEncodedFrameCallback, MarksTimingFramesPeriodicallyTogether) { + const int64_t kDelayMs = 29; + const size_t kMinFrameSize = 10; + const size_t kMaxFrameSize = 20; + const int kNumFrames = 1000; + const int kNumStreams = 3; + // No outliers as 1000 is larger than anything from range [10,20]. + const std::vector kAverageSize = {1000, 1000, 1000}; + auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize, + kAverageSize, kNumStreams, kNumFrames); + // Timing frames should be tirggered every delayMs. + // As no outliers are expected, frames on all streams have to be + // marked together. + int last_timing_frame = -1; + for (int i = 0; i < kNumFrames; ++i) { + int num_normal = 0; + int num_timing = 0; + int num_dropped = 0; + for (int s = 0; s < kNumStreams; ++s) { + if (frames[s][i] == FrameType::kTiming) { + ++num_timing; + } else if (frames[s][i] == FrameType::kNormal) { + ++num_normal; + } else { + ++num_dropped; + } + } + // Can't have both normal and timing frames at the same timstamp. + EXPECT_TRUE(num_timing == 0 || num_normal == 0); + if (num_dropped < kNumStreams) { + if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) { + // If didn't have timing frames for a period, current sent frame has to + // be one. No normal frames should be sent. + EXPECT_EQ(num_normal, 0); + } else { + // No unneeded timing frames should be sent. + EXPECT_EQ(num_timing, 0); + } + } + if (num_timing > 0) + last_timing_frame = i; + } +} + +TEST(TestVCMEncodedFrameCallback, MarksOutliers) { + const int64_t kDelayMs = 29; + const size_t kMinFrameSize = 2495; + const size_t kMaxFrameSize = 2505; + const int kNumFrames = 1000; + const int kNumStreams = 3; + // Possible outliers as 1000 lies in range [995, 1005]. + const std::vector kAverageSize = {998, 1000, 1004}; + auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize, + kAverageSize, kNumStreams, kNumFrames); + // All outliers should be marked. + for (int i = 0; i < kNumFrames; ++i) { + for (int s = 0; s < kNumStreams; ++s) { + if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >= + kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) { + // Too big frame. May be dropped or timing, but not normal. + EXPECT_NE(frames[s][i], FrameType::kNormal); + } + } + } +} + +} // namespace test +} // namespace webrtc diff --git a/webrtc/modules/video_coding/include/video_coding_defines.h b/webrtc/modules/video_coding/include/video_coding_defines.h index b27bcd9ead..6ed472df5e 100644 --- a/webrtc/modules/video_coding/include/video_coding_defines.h +++ b/webrtc/modules/video_coding/include/video_coding_defines.h @@ -39,7 +39,14 @@ namespace webrtc { #define VCM_NO_FRAME_DECODED -11 #define VCM_NOT_IMPLEMENTED -20 -enum { kDefaultStartBitrateKbps = 300 }; +enum { + kDefaultStartBitrateKbps = 300, + // Timing frames settings. Timing frames are sent every + // |kDefaultTimingFramesDelayMs|, or if the frame is at least + // |kDefaultOutliserFrameSizePercent| in size of average frame. + kDefaultTimingFramesDelayMs = 200, + kDefaultOutlierFrameSizePercent = 250, +}; enum VCMVideoProtection { kProtectionNone, diff --git a/webrtc/modules/video_coding/packet.cc b/webrtc/modules/video_coding/packet.cc index 4d5ba7c5cf..22ef7aeebc 100644 --- a/webrtc/modules/video_coding/packet.cc +++ b/webrtc/modules/video_coding/packet.cc @@ -32,7 +32,8 @@ VCMPacket::VCMPacket() insertStartCode(false), width(0), height(0), - video_header() { + video_header(), + receive_time_ms(0) { video_header.playout_delay = {-1, -1}; } diff --git a/webrtc/modules/video_coding/packet.h b/webrtc/modules/video_coding/packet.h index e2ad09aedc..f31c513f68 100644 --- a/webrtc/modules/video_coding/packet.h +++ b/webrtc/modules/video_coding/packet.h @@ -47,6 +47,8 @@ class VCMPacket { int height; RTPVideoHeader video_header; + int64_t receive_time_ms; + protected: void CopyCodecSpecifics(const RTPVideoHeader& videoHeader); }; diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc index 6e0b806bc8..7218cd57ca 100644 --- a/webrtc/modules/video_coding/video_codec_initializer.cc +++ b/webrtc/modules/video_coding/video_codec_initializer.cc @@ -12,11 +12,12 @@ #include "webrtc/base/basictypes.h" #include "webrtc/base/logging.h" -#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_types.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" #include "webrtc/modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/include/video_coding_defines.h" #include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" #include "webrtc/system_wrappers/include/clock.h" @@ -165,6 +166,8 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; if (video_codec.minBitrate < kEncoderMinBitrateKbps) video_codec.minBitrate = kEncoderMinBitrateKbps; + video_codec.timing_frame_thresholds = {kDefaultTimingFramesDelayMs, + kDefaultOutlierFrameSizePercent}; RTC_DCHECK_LE(streams.size(), kMaxSimulcastStreams); if (video_codec.codecType == kVideoCodecVP9) { // If the vector is empty, bitrates will be configured automatically. diff --git a/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc b/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc index 3f778a68e1..fe69de4589 100644 --- a/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc +++ b/webrtc/sdk/android/src/jni/androidmediaencoder_jni.cc @@ -1057,6 +1057,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { (codec_mode_ == webrtc::VideoCodecMode::kScreensharing) ? webrtc::VideoContentType::SCREENSHARE : webrtc::VideoContentType::UNSPECIFIED; + image->timing_.is_timing_frame = false; image->_frameType = (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta); image->_completeFrame = true; diff --git a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm index 5d68c22235..1d37ebaed6 100644 --- a/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm +++ b/webrtc/sdk/objc/Framework/Classes/VideoToolbox/encoder.mm @@ -720,9 +720,9 @@ void H264VideoToolboxEncoder::OnEncodedFrame( frame.capture_time_ms_ = render_time_ms; frame._timeStamp = timestamp; frame.rotation_ = rotation; - frame.content_type_ = (mode_ == kScreensharing) ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; + frame.timing_.is_timing_frame = false; h264_bitstream_parser_.ParseBitstream(buffer->data(), buffer->size()); h264_bitstream_parser_.GetLastSliceQp(&frame.qp_); diff --git a/webrtc/test/constants.cc b/webrtc/test/constants.cc index a789cc0df6..3faa83a2ee 100644 --- a/webrtc/test/constants.cc +++ b/webrtc/test/constants.cc @@ -18,6 +18,7 @@ const int kAbsSendTimeExtensionId = 7; const int kTransportSequenceNumberExtensionId = 8; const int kVideoRotationExtensionId = 9; const int kVideoContentTypeExtensionId = 10; +const int kVideoTimingExtensionId = 11; } // namespace test } // namespace webrtc diff --git a/webrtc/test/constants.h b/webrtc/test/constants.h index d0f73d0fa9..e41e0dac75 100644 --- a/webrtc/test/constants.h +++ b/webrtc/test/constants.h @@ -16,5 +16,6 @@ extern const int kAbsSendTimeExtensionId; extern const int kTransportSequenceNumberExtensionId; extern const int kVideoRotationExtensionId; extern const int kVideoContentTypeExtensionId; +extern const int kVideoTimingExtensionId; } // namespace test } // namespace webrtc diff --git a/webrtc/test/fake_encoder.cc b/webrtc/test/fake_encoder.cc index fce12c61a8..a76ac131d1 100644 --- a/webrtc/test/fake_encoder.cc +++ b/webrtc/test/fake_encoder.cc @@ -148,6 +148,7 @@ int32_t FakeEncoder::Encode(const VideoFrame& input_image, ? VideoContentType::SCREENSHARE : VideoContentType::UNSPECIFIED; specifics.codec_name = ImplementationName(); + specifics.codecSpecific.generic.simulcast_idx = i; RTC_DCHECK(callback); if (callback->OnEncodedImage(encoded, &specifics, nullptr).error != EncodedImageCallback::Result::OK) { diff --git a/webrtc/test/frame_generator_capturer.cc b/webrtc/test/frame_generator_capturer.cc index 149f7015b4..af54f3171d 100644 --- a/webrtc/test/frame_generator_capturer.cc +++ b/webrtc/test/frame_generator_capturer.cc @@ -159,7 +159,7 @@ void FrameGeneratorCapturer::InsertFrame() { rtc::CritScope cs(&lock_); if (sending_) { VideoFrame* frame = frame_generator_->NextFrame(); - frame->set_timestamp_us(rtc::TimeMicros()); + frame->set_timestamp_us(clock_->TimeInMicroseconds()); frame->set_ntp_time_ms(clock_->CurrentNtpInMilliseconds()); frame->set_rotation(fake_rotation_); if (first_frame_capture_time_ == -1) { diff --git a/webrtc/test/fuzzers/rtp_packet_fuzzer.cc b/webrtc/test/fuzzers/rtp_packet_fuzzer.cc index da6294824a..db41e1a9ba 100644 --- a/webrtc/test/fuzzers/rtp_packet_fuzzer.cc +++ b/webrtc/test/fuzzers/rtp_packet_fuzzer.cc @@ -90,6 +90,10 @@ void FuzzOneInput(const uint8_t* data, size_t size) { VideoContentType content_type; packet.GetExtension(&content_type); break; + case kRtpExtensionVideoTiming: + VideoTiming timing; + packet.GetExtension(&timing); + break; case kRtpExtensionRtpStreamId: { std::string rsid; packet.GetExtension(&rsid); diff --git a/webrtc/video/payload_router.cc b/webrtc/video/payload_router.cc index 52e9d46ebb..12ced89b4e 100644 --- a/webrtc/video/payload_router.cc +++ b/webrtc/video/payload_router.cc @@ -130,6 +130,21 @@ EncodedImageCallback::Result PayloadRouter::OnEncodedImage( CopyCodecSpecific(codec_specific_info, &rtp_video_header); rtp_video_header.rotation = encoded_image.rotation_; rtp_video_header.content_type = encoded_image.content_type_; + if (encoded_image.timing_.is_timing_frame) { + rtp_video_header.video_timing.encode_start_delta_ms = + VideoTiming::GetDeltaCappedMs(encoded_image.capture_time_ms_, + encoded_image.timing_.encode_start_ms); + rtp_video_header.video_timing.encode_finish_delta_ms = + VideoTiming::GetDeltaCappedMs(encoded_image.capture_time_ms_, + encoded_image.timing_.encode_finish_ms); + rtp_video_header.video_timing.packetization_finish_delta_ms = 0; + rtp_video_header.video_timing.pacer_exit_delta_ms = 0; + rtp_video_header.video_timing.network_timstamp_delta_ms = 0; + rtp_video_header.video_timing.network2_timstamp_delta_ms = 0; + rtp_video_header.video_timing.is_timing_frame = true; + } else { + rtp_video_header.video_timing.is_timing_frame = false; + } rtp_video_header.playout_delay = encoded_image.playout_delay_; int stream_index = rtp_video_header.simulcastIdx; diff --git a/webrtc/video/rtp_video_stream_receiver.cc b/webrtc/video/rtp_video_stream_receiver.cc index f1e530d3a7..6a55a3dc2a 100644 --- a/webrtc/video/rtp_video_stream_receiver.cc +++ b/webrtc/video/rtp_video_stream_receiver.cc @@ -10,8 +10,8 @@ #include "webrtc/video/rtp_video_stream_receiver.h" -#include #include +#include #include "webrtc/base/checks.h" #include "webrtc/base/location.h" @@ -239,6 +239,7 @@ int32_t RtpVideoStreamReceiver::OnReceivedPayloadData( VCMPacket packet(payload_data, payload_size, rtp_header_with_ntp); packet.timesNacked = nack_module_ ? nack_module_->OnReceivedPacket(packet) : -1; + packet.receive_time_ms = clock_->TimeInMilliseconds(); // In the case of a video stream without picture ids and no rtx the // RtpFrameReferenceFinder will need to know about padding to @@ -520,6 +521,11 @@ void RtpVideoStreamReceiver::NotifyReceiverOfFecPacket( if (header.extension.hasVideoContentType) { rtp_header.type.Video.content_type = header.extension.videoContentType; } + rtp_header.type.Video.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; + if (header.extension.has_video_timing) { + rtp_header.type.Video.video_timing = header.extension.video_timing; + rtp_header.type.Video.video_timing.is_timing_frame = true; + } rtp_header.type.Video.playout_delay = header.extension.playout_delay; OnReceivedPayloadData(nullptr, 0, &rtp_header); diff --git a/webrtc/video/video_quality_test.cc b/webrtc/video/video_quality_test.cc index d19ecbe361..1ab23ed3a2 100644 --- a/webrtc/video/video_quality_test.cc +++ b/webrtc/video/video_quality_test.cc @@ -1327,6 +1327,8 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, } video_send_config_.rtp.extensions.push_back(RtpExtension( RtpExtension::kVideoContentTypeUri, test::kVideoContentTypeExtensionId)); + video_send_config_.rtp.extensions.push_back(RtpExtension( + RtpExtension::kVideoTimingUri, test::kVideoTimingExtensionId)); video_encoder_config_.min_transmit_bitrate_bps = params_.video.min_transmit_bps; diff --git a/webrtc/video/video_send_stream_tests.cc b/webrtc/video/video_send_stream_tests.cc index 3bb565fc92..db1ba4f490 100644 --- a/webrtc/video/video_send_stream_tests.cc +++ b/webrtc/video/video_send_stream_tests.cc @@ -337,6 +337,40 @@ TEST_F(VideoSendStreamTest, SupportsVideoContentType) { RunBaseTest(&test); } +TEST_F(VideoSendStreamTest, SupportsVideoTimingFrames) { + class VideoRotationObserver : public test::SendTest { + public: + VideoRotationObserver() : SendTest(kDefaultTimeoutMs) { + EXPECT_TRUE(parser_->RegisterRtpHeaderExtension( + kRtpExtensionVideoTiming, test::kVideoTimingExtensionId)); + } + + Action OnSendRtp(const uint8_t* packet, size_t length) override { + RTPHeader header; + EXPECT_TRUE(parser_->Parse(packet, length, &header)); + if (header.extension.has_video_timing) { + observation_complete_.Set(); + } + return SEND_PACKET; + } + + void ModifyVideoConfigs( + VideoSendStream::Config* send_config, + std::vector* receive_configs, + VideoEncoderConfig* encoder_config) override { + send_config->rtp.extensions.clear(); + send_config->rtp.extensions.push_back(RtpExtension( + RtpExtension::kVideoTimingUri, test::kVideoTimingExtensionId)); + } + + void PerformTest() override { + EXPECT_TRUE(Wait()) << "Timed out while waiting for timing frames."; + } + } test; + + RunBaseTest(&test); +} + class FakeReceiveStatistics : public NullReceiveStatistics { public: FakeReceiveStatistics(uint32_t send_ssrc, diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 3308961b61..4761c12311 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -675,7 +675,14 @@ void ViEEncoder::OnFrame(const VideoFrame& video_frame) { VideoFrame incoming_frame = video_frame; // Local time in webrtc time base. - int64_t current_time_ms = clock_->TimeInMilliseconds(); + int64_t current_time_us = clock_->TimeInMicroseconds(); + int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec; + // In some cases, e.g., when the frame from decoder is fed to encoder, + // the timestamp may be set to the future. As the encoding pipeline assumes + // capture time to be less than present time, we should reset the capture + // timestamps here. Otherwise there may be issues with RTP send stream. + if (incoming_frame.timestamp_us() > current_time_us) + incoming_frame.set_timestamp_us(current_time_us); // Capture time may come from clock with an offset and drift from clock_. int64_t capture_ntp_time_ms;